def test_InputUntouched(self):
# Check that PutDirectory doesn't alter its inputs.
with working_directory.TemporaryWorkingDirectory() as work_dir:
temp1 = os.path.join(work_dir, 'temp1')
hashing_tools_test.GenerateTestTree('input_untouched', temp1)
h1 = hashing_tools.StableHashPath(temp1)
self._dir_storage.PutDirectory(temp1, 'hello')
h2 = hashing_tools.StableHashPath(temp1)
self.assertEqual(h1, h2)
def test_WriteRead(self):
# Check that a directory can be written and then read back.
with working_directory.TemporaryWorkingDirectory() as work_dir:
temp1 = os.path.join(work_dir, 'temp1')
temp2 = os.path.join(work_dir, 'temp2')
hashing_tools_test.GenerateTestTree('write_read', temp1)
self._dir_storage.PutDirectory(temp1, 'foo')
self._dir_storage.GetDirectory('foo', temp2)
self.assertEqual(hashing_tools.StableHashPath(temp1),
hashing_tools.StableHashPath(temp2))
def test_Directory(self):
# Check that a directory tree works.
with working_directory.TemporaryWorkingDirectory() as work_dir:
a1 = os.path.join(work_dir, 'a1')
a2 = os.path.join(work_dir, 'a2')
for path in [a1, a2]:
GenerateTestTree('gorp', path)
h1 = hashing_tools.StableHashPath(a1)
h2 = hashing_tools.StableHashPath(a2)
self.assertEqual(h1, h2)
self.assertEqual(40, len(h1))
def test_File(self):
# Check that one file works.
with working_directory.TemporaryWorkingDirectory() as work_dir:
filename1 = os.path.join(work_dir, 'myfile1')
filename2 = os.path.join(work_dir, 'myfile2')
file_tools.WriteFile('booga', filename1)
file_tools.WriteFile('booga', filename2)
h1 = hashing_tools.StableHashPath(filename1)
h2 = hashing_tools.StableHashPath(filename2)
self.assertEqual(h1, h2)
self.assertEqual(40, len(h1))
def BuildSignature(self, package, inputs, commands, hasher=None):
"""Compute a total checksum for a computation.
The computed hash includes system properties, inputs, and the commands run.
Args:
package: The name of the package computed.
inputs: A dict of names -> files/directories to be included in the
inputs set.
commands: A list of command.Command objects describing the commands run
for this computation.
hasher: Optional hasher to use.
Returns:
A hex formatted sha1 to use as a computation key or a human readable
signature.
"""
if hasher is None:
h = hashlib.sha1()
else:
h = hasher
h.update('package:' + package)
h.update('summary:' + self.SystemSummary())
for command in commands:
h.update('command:')
h.update(str(command))
for key in sorted(inputs.keys()):
h.update('item_name:' + key + '\x00')
h.update('item:' + hashing_tools.StableHashPath(inputs[key]))
return h.hexdigest()
def WriteOutputFromHash(self, package, out_hash, output):
"""Write output from the cache.
Args:
package: Package name (for tgz name).
out_hash: Hash of desired output.
output: Output path.
Returns:
URL from which output was obtained if successful, or None if not.
"""
key = self.KeyForOutput(package, out_hash)
url = self._directory_storage.GetDirectory(key, output)
if not url:
logging.debug('Failed to retrieve %s' % key)
return None
if hashing_tools.StableHashPath(output) != out_hash:
logging.warning('Object does not match expected hash, '
'has hashing method changed?')
return None
return url
def WriteResultToCache(self, package, build_signature, output):
"""Cache a computed result by key.
Also prints URLs when appropriate.
Args:
package: Package name (for tgz name).
build_signature: The input hash of the computation.
output: A path containing the output of the computation.
"""
if not self._cache_results:
return
out_hash = hashing_tools.StableHashPath(output)
try:
output_key = self.KeyForOutput(package, out_hash)
# Try to get an existing copy in a temporary directory.
wd = working_directory.TemporaryWorkingDirectory()
with wd as work_dir:
temp_output = os.path.join(work_dir, 'out')
url = self._directory_storage.GetDirectory(
output_key, temp_output)
if url is None:
# Isn't present. Cache the computed result instead.
url = self._directory_storage.PutDirectory(
output, output_key)
logging.info('Computed fresh result and cached it.')
else:
# Cached version is present. Replace the current output with that.
file_tools.RemoveDirectoryIfPresent(output)
shutil.move(temp_output, output)
logging.info('Recomputed result matches cached value, '
'using cached value instead.')
# Upload an entry mapping from computation input to output hash.
self._storage.PutData(out_hash,
self.KeyForBuildSignature(build_signature))
self.PrintDownloadURL(url)
except gsd_storage.GSDStorageError:
logging.info('Failed to cache result.')
raise
def Main(args):
if len(args) != 1:
sys.stderr.write('Usage: %s <path>\n' % sys.argv[0])
sys.exit(1)
print hashing_tools.StableHashPath(args[0])