def test_combined_cache(self):
"""Make sure that the combined cache finds what it should and that it backfills."""
httpd = None
httpd_thread = None
try:
with temporary_dir() as http_root:
with temporary_dir() as cache_root:
with pushd(http_root): # SimpleRESTHandler serves from the cwd.
httpd = SocketServer.TCPServer(('localhost', 0), SimpleRESTHandler)
port = httpd.server_address[1]
httpd_thread = Thread(target=httpd.serve_forever)
httpd_thread.start()
with temporary_dir() as artifact_root:
local = LocalArtifactCache(None, artifact_root, cache_root)
remote = RESTfulArtifactCache(MockLogger(), artifact_root, 'http://localhost:%d' % port)
combined = CombinedArtifactCache([local, remote])
key = CacheKey('muppet_key', 'fake_hash', 42)
with temporary_file(artifact_root) as f:
# Write the file.
f.write(TEST_CONTENT1)
path = f.name
f.close()
# No cache has key.
self.assertFalse(local.has(key))
self.assertFalse(remote.has(key))
self.assertFalse(combined.has(key))
# No cache returns key.
self.assertFalse(bool(local.use_cached_files(key)))
self.assertFalse(bool(remote.use_cached_files(key)))
self.assertFalse(bool(combined.use_cached_files(key)))
# Attempting to use key that no cache had should not change anything.
self.assertFalse(local.has(key))
self.assertFalse(remote.has(key))
self.assertFalse(combined.has(key))
# Add to only remote cache.
remote.insert(key, [path])
self.assertFalse(local.has(key))
self.assertTrue(remote.has(key))
self.assertTrue(combined.has(key))
# Successfully using via remote should NOT change local.
self.assertTrue(bool(remote.use_cached_files(key)))
self.assertFalse(local.has(key))
# Successfully using via combined SHOULD backfill local.
self.assertTrue(bool(combined.use_cached_files(key)))
self.assertTrue(local.has(key))
self.assertTrue(bool(local.use_cached_files(key)))
finally:
if httpd:
httpd.shutdown()
if httpd_thread:
httpd_thread.join()
def create_artifact_cache(log, artifact_root, spec, task_name, action='using'):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list of the above, for a combined cache.
"""
if not spec:
raise ValueError('Empty artifact cache spec')
if isinstance(spec, basestring):
if spec.startswith('/') or spec.startswith('~'):
path = os.path.join(spec, task_name)
log.info('%s %s local artifact cache at %s' % (task_name, action, path))
return LocalArtifactCache(log, artifact_root, path)
elif spec.startswith('http://') or spec.startswith('https://'):
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
pinger = Pinger(timeout=0.5, tries=2)
best_url = select_best_url(spec, pinger, log)
if best_url:
url = best_url.rstrip('/') + '/' + task_name
log.info('%s %s remote artifact cache at %s' % (task_name, action, url))
return RESTfulArtifactCache(log, artifact_root, url)
else:
log.warn('%s has no reachable artifact cache in %s.' % (task_name, spec))
return None
else:
raise ValueError('Invalid artifact cache spec: %s' % spec)
elif isinstance(spec, (list, tuple)):
caches = filter(None, [ create_artifact_cache(log, artifact_root, x, task_name, action) for x in spec ])
return CombinedArtifactCache(caches) if caches else None
def create_local_cache(parent_path):
path = os.path.join(parent_path, self._cache_dirname)
self._log.debug('{0} {1} local artifact cache at {2}'
.format(self._task.stable_name(), action, path))
return LocalArtifactCache(artifact_root, path, compression,
self._options.max_entries_per_target,
permissions=self._options.write_permissions,
dereference=self._options.dereference_symlinks)
def other_machine_cache():
with temporary_dir() as artifact_root:
with temporary_dir() as cache_root:
local_cache = LocalArtifactCache(artifact_root,
cache_root,
compression=1)
yield S3ArtifactCache(artifact_root, 's3://' + _TEST_BUCKET,
local_cache)
def setup_local_cache(self, seperate_extraction_root=False):
with temporary_dir() as artifact_root:
with temporary_dir() as artifact_extraction_root:
with temporary_dir() as cache_root:
extraction_root = (artifact_extraction_root
if seperate_extraction_root else
artifact_root)
yield LocalArtifactCache(artifact_root,
extraction_root,
cache_root,
compression=1)
def setup_local_cache(self):
with temporary_dir() as artifact_root:
with temporary_dir() as cache_root:
yield LocalArtifactCache(artifact_root,
cache_root,
compression=1)
def create_local_cache(parent_path):
path = os.path.join(parent_path, self._stable_name)
self._log.debug('{0} {1} local artifact cache at {2}'.format(
self._stable_name, action, path))
return LocalArtifactCache(artifact_root, path, compression,
self._options.max_entries_per_target)
def create_artifact_cache(log, artifact_root, spec, task_name, compression,
action='using', local=None):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list or tuple of two specs, local, then remote, each as described above
:param log: context.log
:param str artifact_root: The path under which cacheable products will be read/written.
:param str spec: See above.
:param str task_name: The name of the task using this cache (eg 'ScalaCompile')
:param int compression: The gzip compression level for created artifacts.
Valid values are 1-9, or Falsy-y to disable compression.
:param str action: A verb, eg 'read' or 'write' for printed messages.
:param LocalArtifactCache local: A local cache for use by created remote caches
"""
if not spec:
raise EmptyCacheSpecError()
if compression and not isinstance(compression, (int, long)):
raise ValueError('compression value must be an integer: {comp}'.format(comp=compression))
def recurse(new_spec, new_local=local):
return create_artifact_cache(log=log, artifact_root=artifact_root, spec=new_spec,
task_name=task_name, compression=compression, action=action,
local=new_local)
def is_remote(spec):
return spec.startswith('http://') or spec.startswith('https://')
if isinstance(spec, basestring):
if spec.startswith('/') or spec.startswith('~'):
path = os.path.join(spec, task_name)
log.debug('{0} {1} local artifact cache at {2}'.format(task_name, action, path))
return LocalArtifactCache(artifact_root, path, compression)
elif is_remote(spec):
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
pinger = Pinger(timeout=0.5, tries=2)
best_url = select_best_url(spec, pinger, log)
if best_url:
url = best_url.rstrip('/') + '/' + task_name
log.debug('{0} {1} remote artifact cache at {2}'.format(task_name, action, url))
local = local or TempLocalArtifactCache(artifact_root)
return RESTfulArtifactCache(artifact_root, url, local)
else:
log.warn('{0} has no reachable artifact cache in {1}.'.format(task_name, spec))
return None
else:
raise CacheSpecFormatError('Invalid artifact cache spec: {0}'.format(spec))
elif isinstance(spec, (list, tuple)) and len(spec) is 1:
return recurse(spec[0])
elif isinstance(spec, (list, tuple)) and len(spec) is 2:
first = recurse(spec[0])
if not isinstance(first, LocalArtifactCache):
raise LocalCacheSpecRequiredError(
'First of two cache specs must be a local cache path. Found: {0}'.format(spec[0]))
if not is_remote(spec[1]):
raise RemoteCacheSpecRequiredError(
'Second of two cache specs must be a remote spec. Found: {0}'.format(spec[1]))
return recurse(spec[1], new_local=first)
else:
raise InvalidCacheSpecError('Invalid artifact cache spec: {0}'.format(spec))
def test_local_cache(self):
with temporary_dir() as artifact_root:
with temporary_dir() as cache_root:
artifact_cache = LocalArtifactCache(None, artifact_root,
cache_root)
self.do_test_artifact_cache(artifact_cache)
def local_cache(local_artifact_root):
with temporary_dir() as cache_root:
yield LocalArtifactCache(local_artifact_root,
cache_root,
compression=1)
