def test_global_pinger_memo(self):
fast_pinger = Pinger(timeout=self.fast_timeout_seconds, tries=2)
slow_pinger = Pinger(timeout=self.slow_timeout_seconds, tries=2)
self.assertEqual(
fast_pinger.pings([self.slow_netloc])[0][1], Pinger.UNREACHABLE, msg=self.message)
self.assertNotEqual(
slow_pinger.pings([self.slow_netloc])[0][1], Pinger.UNREACHABLE, msg=self.message)
def __init__(self, options, log, stable_name, pinger=None, resolver=None):
"""Create a cache factory from settings.
:param options: Task's scoped options.
:param log: Task's context log.
:param stable_name: Task's stable name.
:param pinger: Pinger to choose the best remote artifact cache URL.
:param resolver: Resolver to look up remote artifact cache URLs.
:return: cache factory.
"""
self._options = options
self._log = log
self._stable_name = stable_name
# Created on-demand.
self._read_cache = None
self._write_cache = None
# Protects local filesystem setup, and assignment to the references above.
self._cache_setup_lock = threading.Lock()
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
# TODO: Make lazy.
self._pinger = pinger or Pinger(timeout=self._options.pinger_timeout,
tries=self._options.pinger_tries)
# resolver is also close but failing to resolve might have broader impact than
# single ping failure, therefore use a higher timeout with more retries.
self._resolver = resolver or \
(RESTfulResolver(timeout=1.0, tries=3) if self._options.resolver == 'rest' else \
NoopResolver())
def create_artifact_cache(log, artifact_root, spec, task_name, action='using'):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list of the above, for a combined cache.
"""
if not spec:
raise ValueError('Empty artifact cache spec')
if isinstance(spec, basestring):
if spec.startswith('/') or spec.startswith('~'):
path = os.path.join(spec, task_name)
log.info('%s %s local artifact cache at %s' % (task_name, action, path))
return LocalArtifactCache(log, artifact_root, path)
elif spec.startswith('http://') or spec.startswith('https://'):
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
pinger = Pinger(timeout=0.5, tries=2)
best_url = select_best_url(spec, pinger, log)
if best_url:
url = best_url.rstrip('/') + '/' + task_name
log.info('%s %s remote artifact cache at %s' % (task_name, action, url))
return RESTfulArtifactCache(log, artifact_root, url)
else:
log.warn('%s has no reachable artifact cache in %s.' % (task_name, spec))
return None
else:
raise ValueError('Invalid artifact cache spec: %s' % spec)
elif isinstance(spec, (list, tuple)):
caches = filter(None, [ create_artifact_cache(log, artifact_root, x, task_name, action) for x in spec ])
return CombinedArtifactCache(caches) if caches else None
def test_https_external_pinger(self):
# NB(gmalmquist): I spent quite some time trying to spin up an HTTPS server and get it to work
# with this test, but it appears to be more trouble than it's worth. If you're feeling
# ambitious, feel free to give it a try.
pinger = Pinger(timeout=self.slow_delay_seconds, tries=2)
self.assertLess(pinger.ping(self.https_external_netlock),
Pinger.UNREACHABLE)
def test_pinger_timeout_config(self):
test = Pinger(timeout=self.fast_timeout_seconds, tries=2)
netlocs = [self.fast_netloc, self.slow_netloc]
ping_results = dict(test.pings(netlocs))
self.assertLess(ping_results[self.fast_netloc], self.fast_timeout_seconds)
self.assertEqual(
ping_results[self.slow_netloc], Pinger.UNREACHABLE, msg=self.message)
def test_pinger_times_correct(self): test = Pinger(timeout=self.slow_timeout_seconds, tries=2) netlocs = [self.fast_netloc, self.slow_netloc, self.unreachable_netloc] ping_results = dict(test.pings(netlocs)) self.assertNotEqual(ping_results[self.slow_netloc], Pinger.UNREACHABLE) self.assertLess(ping_results[self.fast_netloc], ping_results[self.slow_netloc]) self.assertEqual(ping_results[self.unreachable_netloc], Pinger.UNREACHABLE, msg=self.message)
def test_pinger_times_correct(self):
test = Pinger(timeout=.5, tries=2)
netlocs = [self.fast_netloc, self.slow_netloc, self.timeout_netloc]
ping_results = dict(test.pings(netlocs))
self.assertLess(ping_results[self.fast_netloc],
ping_results[self.slow_netloc])
self.assertEqual(ping_results[self.timeout_netloc], Pinger.UNREACHABLE)
def pinger(self, timeout, urls, tries=2):
with mock.patch('pants.cache.pinger.Timer.elapsed', new_callable=mock.PropertyMock) as elapsed:
times = []
for url in urls:
for _ in range(tries):
self.expect_response(url, timeout, times)
elapsed.side_effect = times
yield Pinger(timeout=timeout, tries=tries)
# Ensure our mock Timer was used exactly the number of times we expected.
self.assertEqual(elapsed.call_count, len(times))
def __init__(self, options, log, stable_name, pinger=None):
self._options = options
self._log = log
self._stable_name = stable_name
# Created on-demand.
self._read_cache = None
self._write_cache = None
# Protects local filesystem setup, and assignment to the references above.
self._cache_setup_lock = threading.Lock()
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
# TODO: Make lazy.
self._pinger = pinger or Pinger(timeout=0.5, tries=2)
def create_artifact_cache(log, artifact_root, spec, task_name, compression,
action='using', local=None):
"""Returns an artifact cache for the specified spec.
spec can be:
- a path to a file-based cache root.
- a URL of a RESTful cache root.
- a bar-separated list of URLs, where we'll pick the one with the best ping times.
- A list or tuple of two specs, local, then remote, each as described above
:param log: context.log
:param str artifact_root: The path under which cacheable products will be read/written.
:param str spec: See above.
:param str task_name: The name of the task using this cache (eg 'ScalaCompile')
:param int compression: The gzip compression level for created artifacts.
Valid values are 1-9, or Falsy-y to disable compression.
:param str action: A verb, eg 'read' or 'write' for printed messages.
:param LocalArtifactCache local: A local cache for use by created remote caches
"""
if not spec:
raise EmptyCacheSpecError()
if compression and not isinstance(compression, (int, long)):
raise ValueError('compression value must be an integer: {comp}'.format(comp=compression))
def recurse(new_spec, new_local=local):
return create_artifact_cache(log=log, artifact_root=artifact_root, spec=new_spec,
task_name=task_name, compression=compression, action=action,
local=new_local)
def is_remote(spec):
return spec.startswith('http://') or spec.startswith('https://')
if isinstance(spec, basestring):
if spec.startswith('/') or spec.startswith('~'):
path = os.path.join(spec, task_name)
log.debug('{0} {1} local artifact cache at {2}'.format(task_name, action, path))
return LocalArtifactCache(artifact_root, path, compression)
elif is_remote(spec):
# Caches are supposed to be close, and we don't want to waste time pinging on no-op builds.
# So we ping twice with a short timeout.
pinger = Pinger(timeout=0.5, tries=2)
best_url = select_best_url(spec, pinger, log)
if best_url:
url = best_url.rstrip('/') + '/' + task_name
log.debug('{0} {1} remote artifact cache at {2}'.format(task_name, action, url))
local = local or TempLocalArtifactCache(artifact_root)
return RESTfulArtifactCache(artifact_root, url, local)
else:
log.warn('{0} has no reachable artifact cache in {1}.'.format(task_name, spec))
return None
else:
raise CacheSpecFormatError('Invalid artifact cache spec: {0}'.format(spec))
elif isinstance(spec, (list, tuple)) and len(spec) is 1:
return recurse(spec[0])
elif isinstance(spec, (list, tuple)) and len(spec) is 2:
first = recurse(spec[0])
if not isinstance(first, LocalArtifactCache):
raise LocalCacheSpecRequiredError(
'First of two cache specs must be a local cache path. Found: {0}'.format(spec[0]))
if not is_remote(spec[1]):
raise RemoteCacheSpecRequiredError(
'Second of two cache specs must be a remote spec. Found: {0}'.format(spec[1]))
return recurse(spec[1], new_local=first)
else:
raise InvalidCacheSpecError('Invalid artifact cache spec: {0}'.format(spec))
