def test_HitsCacheSecondTime(self): # Test that the computation is not performed on a second instance. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('HitsCacheSecondTime', work_dir) self._tally = 0 def Copy(subst, src, dst): self._tally += 1 shutil.copyfile(subst.SubstituteAbsPaths(src), subst.SubstituteAbsPaths(dst)) self._url = None def stash_url(urls): self._url = urls o = once.Once(storage=fake_storage.FakeStorage(), print_url=stash_url, system_summary='test') o.Run('test', self._input_dirs, self._output_dirs[0], [command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')]) initial_url = self._url self._url = None o.Run('test', self._input_dirs, self._output_dirs[1], [command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')]) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0])) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[1])) self.assertEquals(1, self._tally) self.assertEquals(initial_url, self._url)
def test_UseCachedResultsFalse(self): # Check that the use_cached_results=False does indeed cause computations # to be redone, even when present in the cache. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('UseCachedResultsFalse', work_dir) self._tally = 0 def check_call(cmd, **kwargs): subprocess.check_call(cmd, **kwargs) self._tally += 1 o = once.Once(storage=fake_storage.FakeStorage(), use_cached_results=False, check_call=check_call) o.Run('test', self._input_dirs, self._output_dirs[0], [ command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) o.Run('test', self._input_dirs, self._output_dirs[1], [ command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) self.assertEquals(2, self._tally) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0])) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[1]))
def test_HitsCacheSecondTime(self): # Test that the computation is not performed on a second instance. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('HitsCacheSecondTime', work_dir) self._tally = 0 def check_call(cmd, **kwargs): self._tally += 1 subprocess.check_call(cmd, **kwargs) self._url = None def stash_url(urls): self._url = urls o = once.Once(storage=fake_storage.FakeStorage(), check_call=check_call, print_url=stash_url) o.Run('test', self._input_dirs, self._output_dirs[0], [ command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) initial_url = self._url self._url = None o.Run('test', self._input_dirs, self._output_dirs[1], [ command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0])) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[1])) self.assertEquals(1, self._tally) self.assertEquals(initial_url, self._url)
def test_BadRead(self): # Check that reading from a non-existant key, fails. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) self.assertEquals(None, storage.GetData('foo'))
def test_HitWrappedStorage(self): # Check that if something isn't locally cached primary storage is hit. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) mem_storage.PutData('hello', 'foo') self.assertEquals('hello', storage.GetData('foo'))
def test_AcceptSlashesAndDots(self): # Check that keys with slashes and dots are okay. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) storage.PutData('hello', 'this/is/a/cool_test.txt') self.assertEquals('hello', storage.GetData('this/is/a/cool_test.txt'))
def test_Exists(self): # Checks that exists works properly. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) storage.PutData('bar', 'foo') self.assertTrue(storage.Exists('foo')) self.assertFalse(storage.Exists('bad_foo'))
def test_WriteRead(self): # Check that things written with PutData can be read back. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) storage.PutData('bar', 'foo') self.CanBeReadBothWays(storage, 'foo', os.path.join(work_dir, 'out'), 'bar')
def test_HitLocalFirst(self): # Check that reading hits local storage first. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) storage.PutData('there', 'foo') mem_storage.PutData('hello', 'foo') self.assertEquals('there', storage.GetData('foo'))
def test_FirstTime(self): # Test that the computation is always performed if the cache is empty. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('FirstTime', work_dir) o = once.Once(storage=fake_storage.FakeStorage(), system_summary='test') o.Run('test', self._input_dirs, self._output_dirs[0], [command.Copy('%(input0)s/in0', '%(output)s/out')]) self.assertEquals('FirstTimedata0', file_tools.ReadFile(self._output_files[0]))
def test_Mkdir(self): # Test the Mkdir convenience wrapper works. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('Mkdir', work_dir) foo = os.path.join(work_dir, 'foo') o = once.Once(storage=fake_storage.FakeStorage(), cache_results=False, system_summary='test') o.Run('test', self._input_dirs, foo, [command.Mkdir('%(output)s/hi')]) self.assertTrue(os.path.isdir(os.path.join(foo, 'hi')))
def test_WriteOnlyToLocal(self): # Check that things written hit local storage, not the network. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) storage.PutData('bar', 'foo') self.assertEquals(None, mem_storage.GetData('foo')) bar = os.path.join(work_dir, 'bar_file') file_tools.WriteFile('bar', bar) storage.PutFile(bar, 'foo') self.assertEquals(None, mem_storage.GetData('foo'))
def test_InvalidKey(self): # Check that an invalid key asserts. with working_directory.TemporaryWorkingDirectory() as work_dir: mem_storage = fake_storage.FakeStorage() storage = local_storage_cache.LocalStorageCache( cache_path=os.path.join(work_dir, 'db'), storage=mem_storage) bar = os.path.join(work_dir, 'bar_file') file_tools.WriteFile('bar', bar) self.assertRaises(KeyError, storage.PutData, 'bar', 'foo$') self.assertRaises(KeyError, storage.GetData, 'foo^') self.assertRaises(KeyError, storage.PutFile, bar, 'foo#') self.assertRaises(KeyError, storage.GetFile, 'foo!', 'bar')
def test_NumCores(self): # Test that the core count is substituted. Since we don't know how many # cores the test machine will have, just check that it's an integer. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('NumCores', work_dir) o = once.Once(storage=fake_storage.FakeStorage(), system_summary='test') def CheckCores(subst): self.assertNotEquals(0, int(subst.Substitute('%(cores)s'))) o.Run('test', {}, self._output_dirs[0], [command.Runnable(CheckCores)])
def test_CacheResultsFalse(self): # Check that setting cache_results=False prevents results from being written # to the cache. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('CacheResultsFalse', work_dir) storage = fake_storage.FakeStorage() o = once.Once(storage=storage, cache_results=False) o.Run('test', self._input_dirs, self._output_dirs[0], [ command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) self.assertEquals(0, storage.ItemCount()) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0]))
def test_Command(self): # Test a plain command. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('Command', work_dir) o = once.Once(storage=fake_storage.FakeStorage(), system_summary='test') o.Run('test', self._input_dirs, self._output_dirs[0], [ command.Command([ sys.executable, '-c', 'import sys; open(sys.argv[1], "wb").write("hello")', '%(output)s/out' ]) ]) self.assertEquals('hello', file_tools.ReadFile(self._output_files[0]))
def test_RecomputeHashMatches(self): # Test that things don't get stored to the output cache if they exist # already. with working_directory.TemporaryWorkingDirectory() as work_dir: # Setup test data in input0, input1 using memory storage. self.GenerateTestData('RecomputeHashMatches', work_dir) fs = fake_storage.FakeStorage() ds = directory_storage.DirectoryStorageAdapter(storage=fs) o = once.Once(storage=fs) # Run the computation (compute the length of a file) from input0 to # output0. o.Run('test', self._input_dirs, self._output_dirs[0], [ self.FileLength( '%(input0)s/in0', '%(output)s/out', cwd=work_dir) ]) # Check that 2 writes have occurred. One to write a mapping from in->out, # and one for the output data. self.assertEquals(2, fs.WriteCount()) # Run the computation again from input1 to output1. # (These should have the same length.) o.Run('test', self._input_dirs, self._output_dirs[1], [ self.FileLength( '%(input1)s/in1', '%(output)s/out', cwd=work_dir) ]) # Write count goes up by one as an in->out hash is added, # but no new output is stored (as it is the same). self.assertEquals(3, fs.WriteCount()) # Check that the test is still valid: # - in0 and in1 have equal length. # - out0 and out1 have that length in them. # - out0 and out1 agree. self.assertEquals( str(len(file_tools.ReadFile(self._input_files[0]))), file_tools.ReadFile(self._output_files[0])) self.assertEquals( str(len(file_tools.ReadFile(self._input_files[1]))), file_tools.ReadFile(self._output_files[1])) self.assertEquals(file_tools.ReadFile(self._output_files[0]), file_tools.ReadFile(self._output_files[1]))
def test_UnpackCommands(self): # Test that unpack commnds get run first and hashed_inputs get # used when present. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('UnpackCommands', work_dir) self._tally = 0 def check_call(cmd, **kwargs): self._tally += 1 subprocess.check_call(cmd, **kwargs) o = once.Once(storage=fake_storage.FakeStorage(), check_call=check_call) alt_inputs = {'input0': os.path.join(work_dir, 'alt_input')} unpack_commands = [ command.Copy('%(input0)s/in0', alt_inputs['input0']) ] commands = [ command.Copy('%(input0)s', '%(output)s/out', cwd=work_dir) ] o.Run('test', self._input_dirs, self._output_dirs[0], commands=commands, unpack_commands=unpack_commands, hashed_inputs=alt_inputs) o.Run('test', self._input_dirs, self._output_dirs[1], commands=commands, unpack_commands=unpack_commands, hashed_inputs=alt_inputs) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0])) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[1])) self.assertEquals(3, self._tally)
def test_UseCachedResultsFalse(self): # Check that the use_cached_results=False does indeed cause computations # to be redone, even when present in the cache. with working_directory.TemporaryWorkingDirectory() as work_dir: self.GenerateTestData('UseCachedResultsFalse', work_dir) self._tally = 0 def Copy(subst, src, dst): self._tally += 1 shutil.copyfile(subst.SubstituteAbsPaths(src), subst.SubstituteAbsPaths(dst)) o = once.Once(storage=fake_storage.FakeStorage(), use_cached_results=False, system_summary='test') o.Run('test', self._input_dirs, self._output_dirs[0], [command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')]) o.Run('test', self._input_dirs, self._output_dirs[1], [command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')]) self.assertEquals(2, self._tally) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[0])) self.assertEquals(file_tools.ReadFile(self._input_files[0]), file_tools.ReadFile(self._output_files[1]))
def setUp(self): storage = fake_storage.FakeStorage() self._dir_storage = directory_storage.DirectoryStorageAdapter(storage)