def test_HitsCacheSecondTime(self):
# Test that the computation is not performed on a second instance.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('HitsCacheSecondTime', work_dir)
self._tally = 0
def Copy(subst, src, dst):
self._tally += 1
shutil.copyfile(subst.SubstituteAbsPaths(src),
subst.SubstituteAbsPaths(dst))
self._url = None
def stash_url(urls):
self._url = urls
o = once.Once(storage=fake_storage.FakeStorage(),
print_url=stash_url,
system_summary='test')
o.Run('test', self._input_dirs, self._output_dirs[0],
[command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')])
initial_url = self._url
self._url = None
o.Run('test', self._input_dirs, self._output_dirs[1],
[command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')])
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[1]))
self.assertEquals(1, self._tally)
self.assertEquals(initial_url, self._url)
def test_UseCachedResultsFalse(self):
# Check that the use_cached_results=False does indeed cause computations
# to be redone, even when present in the cache.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('UseCachedResultsFalse', work_dir)
self._tally = 0
def check_call(cmd, **kwargs):
subprocess.check_call(cmd, **kwargs)
self._tally += 1
o = once.Once(storage=fake_storage.FakeStorage(),
use_cached_results=False,
check_call=check_call)
o.Run('test', self._input_dirs, self._output_dirs[0], [
command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
o.Run('test', self._input_dirs, self._output_dirs[1], [
command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
self.assertEquals(2, self._tally)
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[1]))
def test_HitsCacheSecondTime(self):
# Test that the computation is not performed on a second instance.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('HitsCacheSecondTime', work_dir)
self._tally = 0
def check_call(cmd, **kwargs):
self._tally += 1
subprocess.check_call(cmd, **kwargs)
self._url = None
def stash_url(urls):
self._url = urls
o = once.Once(storage=fake_storage.FakeStorage(),
check_call=check_call,
print_url=stash_url)
o.Run('test', self._input_dirs, self._output_dirs[0], [
command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
initial_url = self._url
self._url = None
o.Run('test', self._input_dirs, self._output_dirs[1], [
command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[1]))
self.assertEquals(1, self._tally)
self.assertEquals(initial_url, self._url)
def test_BadRead(self):
# Check that reading from a non-existant key, fails.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
self.assertEquals(None, storage.GetData('foo'))
def test_HitWrappedStorage(self):
# Check that if something isn't locally cached primary storage is hit.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
mem_storage.PutData('hello', 'foo')
self.assertEquals('hello', storage.GetData('foo'))
def test_AcceptSlashesAndDots(self):
# Check that keys with slashes and dots are okay.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('hello', 'this/is/a/cool_test.txt')
self.assertEquals('hello',
storage.GetData('this/is/a/cool_test.txt'))
def test_Exists(self):
# Checks that exists works properly.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('bar', 'foo')
self.assertTrue(storage.Exists('foo'))
self.assertFalse(storage.Exists('bad_foo'))
def test_WriteRead(self):
# Check that things written with PutData can be read back.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('bar', 'foo')
self.CanBeReadBothWays(storage, 'foo',
os.path.join(work_dir, 'out'), 'bar')
def test_HitLocalFirst(self):
# Check that reading hits local storage first.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('there', 'foo')
mem_storage.PutData('hello', 'foo')
self.assertEquals('there', storage.GetData('foo'))
def test_FirstTime(self):
# Test that the computation is always performed if the cache is empty.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('FirstTime', work_dir)
o = once.Once(storage=fake_storage.FakeStorage(),
system_summary='test')
o.Run('test', self._input_dirs, self._output_dirs[0],
[command.Copy('%(input0)s/in0', '%(output)s/out')])
self.assertEquals('FirstTimedata0',
file_tools.ReadFile(self._output_files[0]))
def test_Mkdir(self):
# Test the Mkdir convenience wrapper works.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('Mkdir', work_dir)
foo = os.path.join(work_dir, 'foo')
o = once.Once(storage=fake_storage.FakeStorage(),
cache_results=False,
system_summary='test')
o.Run('test', self._input_dirs, foo,
[command.Mkdir('%(output)s/hi')])
self.assertTrue(os.path.isdir(os.path.join(foo, 'hi')))
def test_WriteOnlyToLocal(self):
# Check that things written hit local storage, not the network.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
storage.PutData('bar', 'foo')
self.assertEquals(None, mem_storage.GetData('foo'))
bar = os.path.join(work_dir, 'bar_file')
file_tools.WriteFile('bar', bar)
storage.PutFile(bar, 'foo')
self.assertEquals(None, mem_storage.GetData('foo'))
def test_InvalidKey(self):
# Check that an invalid key asserts.
with working_directory.TemporaryWorkingDirectory() as work_dir:
mem_storage = fake_storage.FakeStorage()
storage = local_storage_cache.LocalStorageCache(
cache_path=os.path.join(work_dir, 'db'), storage=mem_storage)
bar = os.path.join(work_dir, 'bar_file')
file_tools.WriteFile('bar', bar)
self.assertRaises(KeyError, storage.PutData, 'bar', 'foo$')
self.assertRaises(KeyError, storage.GetData, 'foo^')
self.assertRaises(KeyError, storage.PutFile, bar, 'foo#')
self.assertRaises(KeyError, storage.GetFile, 'foo!', 'bar')
def test_NumCores(self):
# Test that the core count is substituted. Since we don't know how many
# cores the test machine will have, just check that it's an integer.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('NumCores', work_dir)
o = once.Once(storage=fake_storage.FakeStorage(),
system_summary='test')
def CheckCores(subst):
self.assertNotEquals(0, int(subst.Substitute('%(cores)s')))
o.Run('test', {}, self._output_dirs[0],
[command.Runnable(CheckCores)])
def test_CacheResultsFalse(self):
# Check that setting cache_results=False prevents results from being written
# to the cache.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('CacheResultsFalse', work_dir)
storage = fake_storage.FakeStorage()
o = once.Once(storage=storage, cache_results=False)
o.Run('test', self._input_dirs, self._output_dirs[0], [
command.Copy('%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
self.assertEquals(0, storage.ItemCount())
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
def test_Command(self):
# Test a plain command.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('Command', work_dir)
o = once.Once(storage=fake_storage.FakeStorage(),
system_summary='test')
o.Run('test', self._input_dirs, self._output_dirs[0], [
command.Command([
sys.executable, '-c',
'import sys; open(sys.argv[1], "wb").write("hello")',
'%(output)s/out'
])
])
self.assertEquals('hello',
file_tools.ReadFile(self._output_files[0]))
def test_RecomputeHashMatches(self):
# Test that things don't get stored to the output cache if they exist
# already.
with working_directory.TemporaryWorkingDirectory() as work_dir:
# Setup test data in input0, input1 using memory storage.
self.GenerateTestData('RecomputeHashMatches', work_dir)
fs = fake_storage.FakeStorage()
ds = directory_storage.DirectoryStorageAdapter(storage=fs)
o = once.Once(storage=fs)
# Run the computation (compute the length of a file) from input0 to
# output0.
o.Run('test', self._input_dirs, self._output_dirs[0], [
self.FileLength(
'%(input0)s/in0', '%(output)s/out', cwd=work_dir)
])
# Check that 2 writes have occurred. One to write a mapping from in->out,
# and one for the output data.
self.assertEquals(2, fs.WriteCount())
# Run the computation again from input1 to output1.
# (These should have the same length.)
o.Run('test', self._input_dirs, self._output_dirs[1], [
self.FileLength(
'%(input1)s/in1', '%(output)s/out', cwd=work_dir)
])
# Write count goes up by one as an in->out hash is added,
# but no new output is stored (as it is the same).
self.assertEquals(3, fs.WriteCount())
# Check that the test is still valid:
# - in0 and in1 have equal length.
# - out0 and out1 have that length in them.
# - out0 and out1 agree.
self.assertEquals(
str(len(file_tools.ReadFile(self._input_files[0]))),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(
str(len(file_tools.ReadFile(self._input_files[1]))),
file_tools.ReadFile(self._output_files[1]))
self.assertEquals(file_tools.ReadFile(self._output_files[0]),
file_tools.ReadFile(self._output_files[1]))
def test_UnpackCommands(self):
# Test that unpack commnds get run first and hashed_inputs get
# used when present.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('UnpackCommands', work_dir)
self._tally = 0
def check_call(cmd, **kwargs):
self._tally += 1
subprocess.check_call(cmd, **kwargs)
o = once.Once(storage=fake_storage.FakeStorage(),
check_call=check_call)
alt_inputs = {'input0': os.path.join(work_dir, 'alt_input')}
unpack_commands = [
command.Copy('%(input0)s/in0', alt_inputs['input0'])
]
commands = [
command.Copy('%(input0)s', '%(output)s/out', cwd=work_dir)
]
o.Run('test',
self._input_dirs,
self._output_dirs[0],
commands=commands,
unpack_commands=unpack_commands,
hashed_inputs=alt_inputs)
o.Run('test',
self._input_dirs,
self._output_dirs[1],
commands=commands,
unpack_commands=unpack_commands,
hashed_inputs=alt_inputs)
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[1]))
self.assertEquals(3, self._tally)
def test_UseCachedResultsFalse(self):
# Check that the use_cached_results=False does indeed cause computations
# to be redone, even when present in the cache.
with working_directory.TemporaryWorkingDirectory() as work_dir:
self.GenerateTestData('UseCachedResultsFalse', work_dir)
self._tally = 0
def Copy(subst, src, dst):
self._tally += 1
shutil.copyfile(subst.SubstituteAbsPaths(src),
subst.SubstituteAbsPaths(dst))
o = once.Once(storage=fake_storage.FakeStorage(),
use_cached_results=False,
system_summary='test')
o.Run('test', self._input_dirs, self._output_dirs[0],
[command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')])
o.Run('test', self._input_dirs, self._output_dirs[1],
[command.Runnable(Copy, '%(input0)s/in0', '%(output)s/out')])
self.assertEquals(2, self._tally)
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[0]))
self.assertEquals(file_tools.ReadFile(self._input_files[0]),
file_tools.ReadFile(self._output_files[1]))
def setUp(self):
storage = fake_storage.FakeStorage()
self._dir_storage = directory_storage.DirectoryStorageAdapter(storage)
