def test_support_url_multi(self):
"""Tests to make sure existing base urls function as expected."""
with temporary_dir() as invalid_local_files, temporary_dir(
) as valid_local_files:
binary_util = BinaryUtil(baseurls=[
'BLATANTLY INVALID URL',
'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
invalid_local_files,
valid_local_files,
'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
],
timeout_secs=30,
bootstrapdir='/tmp')
binary_path = binary_util._select_binary_base_path(
supportdir='bin/protobuf', version='2.4.1', name='protoc')
contents = b'proof'
with safe_open(os.path.join(valid_local_files, binary_path),
'wb') as fp:
fp.write(contents)
with binary_util._select_binary_stream(
name='protoc', binary_path=binary_path) as stream:
self.assertEqual(contents, stream())
def create(cls, bootstrap_options):
"""
:param Options bootstrap_options: The bootstrap options bag.
"""
binary_tool_fetcher = BinaryToolFetcher(
bootstrap_dir=bootstrap_options.pants_bootstrapdir,
timeout_secs=bootstrap_options.binaries_fetch_timeout_secs,
)
binary_util = BinaryUtil(
baseurls=bootstrap_options.binaries_baseurls,
binary_tool_fetcher=binary_tool_fetcher,
path_by_id=bootstrap_options.binaries_path_by_id,
allow_external_binary_tool_downloads=bootstrap_options.
allow_external_binary_tool_downloads,
)
return WatchmanLauncher(
binary_util,
bootstrap_options.level,
bootstrap_options.watchman_version,
bootstrap_options.watchman_supportdir,
bootstrap_options.watchman_startup_timeout,
bootstrap_options.watchman_socket_timeout,
bootstrap_options.watchman_socket_path,
bootstrap_options.pants_subprocessdir,
)
def test_support_url_fallback(self):
"""Tests fallback behavior with multiple support baseurls.
Mocks up some dummy baseurls and then swaps out the URL reader to make sure urls are accessed
and others are not.
"""
fake_base, fake_url = self._fake_base, self._fake_url
bases = [fake_base('apple'), fake_base('orange'), fake_base('banana')]
binary_util = BinaryUtil(bases, 30, '/tmp')
binaries = {t[2]: t for t in (('bin/protobuf', '2.4.1', 'protoc'),
('bin/ivy', '4.3.7', 'ivy'),
('bin/bash', '4.4.3', 'bash'))}
reader = self.MapReader({
fake_url(binaries, bases[0], 'protoc'): 'SEEN PROTOC',
fake_url(binaries, bases[0], 'ivy'): 'SEEN IVY',
fake_url(binaries, bases[1], 'bash'): 'SEEN BASH',
fake_url(binaries, bases[1], 'protoc'): 'UNSEEN PROTOC 1',
fake_url(binaries, bases[2], 'protoc'): 'UNSEEN PROTOC 2',
fake_url(binaries, bases[2], 'ivy'): 'UNSEEN IVY 2',
})
unseen = [item for item in reader.values() if item.startswith('SEEN ')]
for supportdir, version, name in binaries.values():
with binary_util._select_binary_stream(supportdir=supportdir,
version=version,
name=name,
url_opener=reader) as stream:
self.assertEqual(stream(), 'SEEN ' + name.upper())
unseen.remove(stream())
self.assertEqual(0, len(unseen)) # Make sure we've seen all the SEENs.
def _gen_binary_util(cls, baseurls=[], path_by_id=None, allow_external_binary_tool_downloads=True,
uname_func=None, **kwargs):
return BinaryUtil(
baseurls=baseurls,
binary_tool_fetcher=cls._gen_binary_tool_fetcher(**kwargs),
path_by_id=path_by_id,
allow_external_binary_tool_downloads=allow_external_binary_tool_downloads,
uname_func=uname_func)
def test_nobases(self):
"""Tests exception handling if build support urls are improperly specified."""
binary_util = BinaryUtil(baseurls=[], timeout_secs=30, bootstrapdir='/tmp')
with self.assertRaises(binary_util.NoBaseUrlsError):
with binary_util._select_binary_stream(supportdir='bin/protobuf',
version='2.4.1',
name='protoc'):
self.fail('Expected acquisition of the stream to raise.')
def test_select_binary_base_path_override(self):
binary_util = BinaryUtil([], 0, '/tmp',
{('darwin', '100'): ['skynet', '42']})
def uname_func():
return "darwin", "dontcare1", "100.99", "dontcare2", "t1000"
self.assertEquals("supportdir/skynet/42/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_darwin(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "14.9", "dontcare2", "dontcare3",
self.assertEquals("supportdir/mac/10.10/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_linux(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "linux", "dontcare1", "dontcare2", "dontcare3", "amd64"
self.assertEquals("supportdir/linux/x86_64/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_missing_os(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "vms", "dontcare1", "999.9", "dontcare2", "VAX9"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Pants has no binaries for vms'):
binary_util._select_binary_base_path("supportdir", "name", "version", uname_func=uname_func)
def test_select_binary_base_path_missing_version(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Update --binaries-path-by-id to find binaries for darwin x86_64 999\.9\.'):
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func)
def test_select_binary_base_path_missing_version(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
os_id = ('darwin', '999')
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Update --binaries-path-by-id to find binaries for '
r'{}'.format(re.escape(repr(os_id)))):
binary_util._select_binary_base_path("supportdir", "name", "version", uname_func=uname_func)
def test_timeout(self):
fetcher = mock.create_autospec(Fetcher, spec_set=True)
binary_util = BinaryUtil(baseurls=['http://binaries.example.com'],
timeout_secs=42,
bootstrapdir='/tmp')
self.assertFalse(fetcher.download.called)
with binary_util._select_binary_stream('a-binary', 'a-binary/v1.2/a-binary', fetcher=fetcher):
fetcher.download.assert_called_once_with('http://binaries.example.com/a-binary/v1.2/a-binary',
listener=mock.ANY,
path_or_fd=mock.ANY,
timeout_secs=42)
def create(cls, bootstrap_options):
"""
:param Options bootstrap_options: The bootstrap options bag.
"""
binary_util = BinaryUtil(bootstrap_options.binaries_baseurls,
bootstrap_options.binaries_fetch_timeout_secs,
bootstrap_options.pants_bootstrapdir,
bootstrap_options.binaries_path_by_id)
return WatchmanLauncher(binary_util, bootstrap_options.level,
bootstrap_options.watchman_version,
bootstrap_options.watchman_supportdir,
bootstrap_options.watchman_startup_timeout,
bootstrap_options.watchman_socket_timeout,
bootstrap_options.watchman_socket_path,
bootstrap_options.pants_subprocessdir)
def test_support_url_multi(self):
"""Tests to make sure existing base urls function as expected."""
count = 0
binary_util = BinaryUtil(
baseurls=[
'BLATANTLY INVALID URL',
'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
'https://dl.bintray.com/pantsbuild/bin/build-support',
'https://dl.bintray.com/pantsbuild/bin/build-support', # Test duplicate entry handling.
'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
],
timeout_secs=30,
bootstrapdir='/tmp')
with binary_util._select_binary_stream(supportdir='bin/protobuf',
version='2.4.1',
name='protoc') as stream:
stream()
count += 1
self.assertEqual(count, 1)
def _fake_url(cls, binaries, base, binary_key):
binary_util = BinaryUtil([], 0, '/tmp')
supportdir, version, name = binaries[binary_key]
binary = binary_util._select_binary_base_path(supportdir, version, binary_key)
return '{base}/{binary}'.format(base=base, binary=binary)
