def test_miss_on_disk2(self):
'''
Same as the in-memory miss test except we flush the cache in-between,
at a point at which the entry is already invalid. The invalidity is not
actually detected then, but the later lookup should still miss.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Now cause the entry to be invalid by modifying inputs.
with open(input, 'wt') as f:
f.write('bar foo')
# Flush the (now invalid) entry to disk.
c.flush()
# Ensure we miss when now performing a lookup.
output = c.load(['arg1', 'arg2'], cwd)
self.assertIsNone(output)
def test_timestamps(self):
'''
Ensure that modifying a timestamp on one of the inputs has no effect.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Bump the timestamps on the input.
st = os.stat(input)
os.utime(input, (st[stat.ST_ATIME] + 3600, st[stat.ST_MTIME] + 3600))
# Ensure we can find what we just saved.
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
# And after a flush.
c.flush()
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
def test_cache_hit_truncate(self):
'''
A previous accelerator bug resulted in the output file not being
truncated before the accelerator wrote to it. The result was that an
output resulting from a cache hit that was delivered via the
accelerator would have trailing garbage if it was shorter than the
existing file content. More specifically, the following sequence could
occur:
1. Build with cache A enabled and "short string" is output and
cached;
2. Build with different config and "a slightly longer string" is
output (and cached);
3. Build with original config and the accelerator enabled and "short
string" is retrieved, but written without truncating the output.
As a result, the final file content would end up as "short stringonger
string". This test validates that this problem has not been
reintroduced.
'''
root = self.mkdtemp()
internal_root = os.path.join(root, version(), 'cachea')
c = Cache(internal_root)
# Setup a basic, single-input entry.
input1 = self.mkstemp()
with open(input1, 'wt') as f:
f.write('hello world')
inputs = prime_inputs([input1])
cwd = self.mkdtemp()
output = self.mkstemp()
args = ['--cache-dir', root, '--outfile', output]
# Write the entry to the cache with a specific short value.
content = 'moo cow'
c.save(args[:-2], cwd, content, inputs)
c.flush()
del c
# Now write something *longer* into the output file.
with open(output, 'wt') as f:
f.write('some lengthier text')
# Now run the accelerator to retrieve the original, shorter output.
ret, stdout, stderr = self.execute([self.accelerator] + args, cwd=cwd)
# It should have hit the cache and written the correct, shorter output.
self.assertEqual(ret, 0)
self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
with open(output) as f:
data = f.read()
self.assertEqual(data, content)
def test_basic_valgrind(self):
root = self.mkdtemp()
# CAmkES internally suffixes the root with a couple of things to
# namespace the cache.
internal_root = os.path.join(root, version(), 'cachea')
c = Cache(internal_root)
# Construct some fake inputs.
input1 = self.mkstemp()
with open(input1, 'wt') as f:
f.write('hello world')
input2 = self.mkstemp()
with open(input2, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input1, input2])
# And a fake working directory.
cwd = self.mkdtemp()
# Imagine we were saving the output from the following file.
output = self.mkstemp()
# So the command line arguments would be:
args = ['--cache-dir', root, '--outfile', output]
# Save the entry. Note that we truncate the args because the runner and
# the accelerator strip --outfile arguments before interacting with the
# cache.
c.save(args[:-2], cwd, 'moo cow', inputs)
c.flush()
# We're done with the native cache.
del c
# Now let's try to read back the cache entry from the accelerator.
_, _, stderr = self.execute(VALGRIND + [self.debug_accelerator] + args,
cwd=cwd)
if valgrind_found_leak(stderr):
self.fail('camkes-accelerator %s leaks memory:\n%s' %
(' '.join(args), stderr))
_, _, stderr = self.execute(VALGRIND + [self.accelerator] + args,
cwd=cwd)
if valgrind_found_leak(stderr):
self.fail('camkes-accelerator %s leaks memory (not reproducible '
'in debug mode):\n%s' % (' '.join(args), stderr))
def test_basic(self):
'''
Test we can save and retrieve something (expected case).
'''
root = self.mkdtemp()
# CAmkES internally suffixes the root with a couple of things to
# namespace the cache.
internal_root = os.path.join(root, version(), 'cachea')
c = Cache(internal_root)
# Construct some fake inputs.
input1 = self.mkstemp()
with open(input1, 'wt') as f:
f.write('hello world')
input2 = self.mkstemp()
with open(input2, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input1, input2])
# And a fake working directory.
cwd = self.mkdtemp()
# Imagine we were saving the output from the following file.
output = self.mkstemp()
# So the command line arguments would be:
args = ['--cache-dir', root, '--outfile', output]
# Save the entry. Note that we truncate the args because the runner and
# the accelerator strip --outfile arguments before interacting with the
# cache.
c.save(args[:-2], cwd, 'moo cow', inputs)
c.flush()
# We're done with the native cache.
del c
# Now let's try to read back the cache entry from the accelerator.
ret, _, _ = self.execute([self.accelerator] + args, cwd=cwd)
self.assertEqual(ret, 0)
# If it worked, we should have the output in the expected place.
with open(output, 'rt') as f:
data = f.read()
self.assertEqual(data, 'moo cow')
def test_directory_creation(self):
'''
The cache should be capable of creating necessary subdirectories under
its root.
'''
root = os.path.join(self.mkdtemp(), 'non-existent')
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
c.save(['arg1', 'arg2'], os.getcwd(), 'hello world', inputs)
c.flush()
def test_no_inputs(self):
'''
Ensure we can handle an entry with no inputs.
'''
root = self.mkdtemp()
c = Cache(root)
inputs = prime_inputs([])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Ensure we can find what we just saved.
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
# Ensure it is preserved after a flush.
c.flush()
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
def test_cache_miss_inputs_valgrind(self):
# As for the basic test case...
root = self.mkdtemp()
internal_root = os.path.join(root, version(), 'cachea')
c = Cache(internal_root)
input1 = self.mkstemp()
with open(input1, 'wt') as f:
f.write('hello world')
input2 = self.mkstemp()
with open(input2, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input1, input2])
cwd = self.mkdtemp()
output = self.mkstemp()
args = ['--cache-dir', root, '--outfile', output]
c.save(args[:-2], cwd, 'moo cow', inputs)
c.flush()
del c
# Now let's modify one of the inputs.
with open(input2, 'at') as f:
f.write('foo bar')
_, _, stderr = self.execute(VALGRIND + [self.debug_accelerator] + args,
cwd=cwd)
if valgrind_found_leak(stderr):
self.fail('camkes-accelerator %s leaks memory:\n%s' %
(' '.join(args), stderr))
_, _, stderr = self.execute(VALGRIND + [self.accelerator] + args,
cwd=cwd)
if valgrind_found_leak(stderr):
self.fail('camkes-accelerator %s leaks memory (not reproducible '
'in debug mode):\n%s' % (' '.join(args), stderr))
def test_basic(self):
'''
Test we can look up something we've just saved. Note that this test
will not actually perform an on-disk lookup.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Ensure we can find what we just saved.
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
def test_no_args(self):
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save([], cwd, 'hello world', inputs)
# Ensure we can find what we just saved.
output = c.load([], cwd)
self.assertEqual(output, 'hello world')
# Ensure it is preserved after a flush.
c.flush()
output = c.load([], cwd)
self.assertEqual(output, 'hello world')
def test_cache_miss_inputs(self):
'''
Test that we correctly miss when one of the inputs has changed.
'''
# As for the basic test case...
root = self.mkdtemp()
internal_root = os.path.join(root, version(), 'cachea')
c = Cache(internal_root)
input1 = self.mkstemp()
with open(input1, 'wt') as f:
f.write('hello world')
input2 = self.mkstemp()
with open(input2, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input1, input2])
cwd = self.mkdtemp()
output = self.mkstemp()
args = ['--cache-dir', root, '--outfile', output]
c.save(args[:-2], cwd, 'moo cow', inputs)
c.flush()
del c
# Now let's modify one of the inputs.
with open(input2, 'at') as f:
f.write('foo bar')
ret, stdout, stderr = self.execute([self.accelerator] + args, cwd=cwd)
# It should have missed (== non-zero return value with no output).
self.assertNotEqual(ret, 0)
self.assertEqual(stdout, '')
self.assertEqual(stderr, '')
def test_basic_with_flush(self):
'''
Same as the basic test, but we'll flush in-between to ensure we perform
an on-disk lookup.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
c.flush()
# Ensure we can find what we just saved.
output = c.load(['arg1', 'arg2'], cwd)
self.assertEqual(output, 'hello world')
def test_miss_from_missing_file1(self):
'''
Ensure cache misses from missing files function correctly.
'''
root = self.mkdtemp()
c = Cache(root)
_, input = tempfile.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Now cause the entry to be invalid by deleting its input.
os.remove(input)
# Ensure we miss when now performing a lookup.
output = c.load(['arg1', 'arg2'], cwd)
self.assertIsNone(output)
def test_miss_on_disk1(self):
'''
Same as the in-memory miss test except we flush the cache in-between.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
c.flush()
# Now cause the entry to be invalid by modifying inputs.
with open(input, 'wt') as f:
f.write('bar foo')
# Ensure we miss when now performing a lookup.
output = c.load(['arg1', 'arg2'], cwd)
self.assertIsNone(output)
def test_miss_in_memory(self):
'''
Test that an induced cache miss while the cache entry is still in
memory works correctly.
'''
root = self.mkdtemp()
c = Cache(root)
input = self.mkstemp()
with open(input, 'wt') as f:
f.write('foo bar')
inputs = prime_inputs([input])
cwd = os.getcwd()
c.save(['arg1', 'arg2'], cwd, 'hello world', inputs)
# Now cause the entry to be invalid by modifying inputs.
with open(input, 'wt') as f:
f.write('bar foo')
# Ensure we miss when now performing a lookup.
output = c.load(['arg1', 'arg2'], cwd)
self.assertIsNone(output)
