def test_support_url_fallback(self):
"""Tests fallback behavior with multiple support baseurls.
Mocks up some dummy baseurls and then swaps out the URL reader to make sure urls are accessed
and others are not.
"""
fake_base, fake_url = self._fake_base, self._fake_url
bases = [fake_base('apple'), fake_base('orange'), fake_base('banana')]
binary_util = BinaryUtil(bases, 30, '/tmp')
binaries = {t[2]: t for t in (('bin/protobuf', '2.4.1', 'protoc'),
('bin/ivy', '4.3.7', 'ivy'),
('bin/bash', '4.4.3', 'bash'))}
reader = self.MapReader({
fake_url(binaries, bases[0], 'protoc'): 'SEEN PROTOC',
fake_url(binaries, bases[0], 'ivy'): 'SEEN IVY',
fake_url(binaries, bases[1], 'bash'): 'SEEN BASH',
fake_url(binaries, bases[1], 'protoc'): 'UNSEEN PROTOC 1',
fake_url(binaries, bases[2], 'protoc'): 'UNSEEN PROTOC 2',
fake_url(binaries, bases[2], 'ivy'): 'UNSEEN IVY 2',
})
unseen = [item for item in reader.values() if item.startswith('SEEN ')]
for supportdir, version, name in binaries.values():
with binary_util._select_binary_stream(supportdir=supportdir,
version=version,
name=name,
url_opener=reader) as stream:
self.assertEqual(stream(), 'SEEN ' + name.upper())
unseen.remove(stream())
self.assertEqual(0, len(unseen)) # Make sure we've seen all the SEENs.
def test_nobases(self):
"""Tests exception handling if build support urls are improperly specified."""
binary_util = BinaryUtil(baseurls=[], timeout_secs=30, bootstrapdir='/tmp')
with self.assertRaises(binary_util.NoBaseUrlsError):
with binary_util._select_binary_stream(supportdir='bin/protobuf',
version='2.4.1',
name='protoc'):
self.fail('Expected acquisition of the stream to raise.')
def test_select_binary_base_path_linux(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "linux", "dontcare1", "dontcare2", "dontcare3", "amd64"
self.assertEquals("supportdir/linux/x86_64/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_override(self):
binary_util = BinaryUtil([], 0, '/tmp',
{('darwin', '100'): ['skynet', '42']})
def uname_func():
return "darwin", "dontcare1", "100.99", "dontcare2", "t1000"
self.assertEquals("supportdir/skynet/42/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_missing_os(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "vms", "dontcare1", "999.9", "dontcare2", "VAX9"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Pants has no binaries for vms'):
binary_util._select_binary_base_path("supportdir", "name", "version", uname_func=uname_func)
def test_select_binary_base_path_darwin(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "14.9", "dontcare2", "dontcare3",
self.assertEquals("supportdir/mac/10.10/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_missing_os(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "vms", "dontcare1", "999.9", "dontcare2", "VAX9"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Pants has no binaries for vms'):
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func)
def test_select_binary_base_path_missing_version(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Update --binaries-path-by-id to find binaries for darwin x86_64 999\.9\.'):
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func)
def test_select_binary_base_path_override(self):
binary_util = BinaryUtil([], 0, '/tmp',
{('darwin', '100'): ['skynet', '42']})
def uname_func():
return "darwin", "dontcare1", "100.99", "dontcare2", "t1000"
self.assertEquals("supportdir/skynet/42/name/version",
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func))
def test_select_binary_base_path_missing_version(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Update --binaries-path-by-id to find binaries for darwin x86_64 999\.9\.'):
binary_util._select_binary_base_path("supportdir", "name", "version",
uname_func=uname_func)
def test_nobases(self):
"""Tests exception handling if build support urls are improperly specified."""
binary_util = BinaryUtil(baseurls=[],
timeout_secs=30,
bootstrapdir='/tmp')
with self.assertRaises(binary_util.NoBaseUrlsError):
with binary_util._select_binary_stream(supportdir='bin/protobuf',
version='2.4.1',
name='protoc'):
self.fail('Expected acquisition of the stream to raise.')
def test_select_binary_base_path_missing_version(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "999.9", "dontcare2", "x86_64"
os_id = ('darwin', '999')
with self.assertRaisesRegexp(BinaryUtil.MissingMachineInfo,
r'Update --binaries-path-by-id to find binaries for '
r'{}'.format(re.escape(repr(os_id)))):
binary_util._select_binary_base_path("supportdir", "name", "version", uname_func=uname_func)
def test_select_binary_base_path_linux(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "linux", "dontcare1", "dontcare2", "dontcare3", "amd64"
self.assertEquals(
"supportdir/linux/x86_64/version/name",
binary_util._select_binary_base_path("supportdir",
"version",
"name",
uname_func=uname_func))
def test_timeout(self):
fetcher = mock.create_autospec(Fetcher, spec_set=True)
binary_util = BinaryUtil(baseurls=['http://binaries.example.com'],
timeout_secs=42,
bootstrapdir='/tmp')
self.assertFalse(fetcher.download.called)
with binary_util._select_binary_stream('a-binary', 'a-binary/v1.2/a-binary', fetcher=fetcher):
fetcher.download.assert_called_once_with('http://binaries.example.com/a-binary/v1.2/a-binary',
listener=mock.ANY,
path_or_fd=mock.ANY,
timeout_secs=42)
def test_select_binary_base_path_darwin(self):
binary_util = BinaryUtil([], 0, '/tmp')
def uname_func():
return "darwin", "dontcare1", "14.9", "dontcare2", "dontcare3",
self.assertEquals(
"supportdir/mac/10.10/version/name",
binary_util._select_binary_base_path("supportdir",
"version",
"name",
uname_func=uname_func))
def create(cls, bootstrap_options):
"""
:param Options bootstrap_options: The bootstrap options bag.
"""
binary_tool_fetcher = BinaryToolFetcher(
bootstrap_dir=bootstrap_options.pants_bootstrapdir,
timeout_secs=bootstrap_options.binaries_fetch_timeout_secs,
)
binary_util = BinaryUtil(
baseurls=bootstrap_options.binaries_baseurls,
binary_tool_fetcher=binary_tool_fetcher,
path_by_id=bootstrap_options.binaries_path_by_id,
allow_external_binary_tool_downloads=bootstrap_options.
allow_external_binary_tool_downloads,
)
return WatchmanLauncher(
binary_util,
bootstrap_options.level,
bootstrap_options.watchman_version,
bootstrap_options.watchman_supportdir,
bootstrap_options.watchman_startup_timeout,
bootstrap_options.watchman_socket_timeout,
bootstrap_options.watchman_socket_path,
bootstrap_options.pants_subprocessdir,
)
def _gen_binary_util(cls, baseurls=[], path_by_id=None, allow_external_binary_tool_downloads=True,
uname_func=None, **kwargs):
return BinaryUtil(
baseurls=baseurls,
binary_tool_fetcher=cls._gen_binary_tool_fetcher(**kwargs),
path_by_id=path_by_id,
allow_external_binary_tool_downloads=allow_external_binary_tool_downloads,
uname_func=uname_func)
def test_support_url_fallback(self):
"""Tests fallback behavior with multiple support baseurls.
Mocks up some dummy baseurls and then swaps out the URL reader to make sure urls are accessed
and others are not.
"""
fake_base, fake_url = self._fake_base, self._fake_url
bases = [fake_base('apple'), fake_base('orange'), fake_base('banana')]
binary_util = BinaryUtil(bases, 30, '/tmp')
binaries = {
t[2]: t
for t in (('bin/protobuf', '2.4.1', 'protoc'),
('bin/ivy', '4.3.7', 'ivy'), ('bin/bash', '4.4.3',
'bash'))
}
fetcher = self.MapFetcher({
fake_url(binaries, bases[0], 'protoc'):
'SEEN PROTOC',
fake_url(binaries, bases[0], 'ivy'):
'SEEN IVY',
fake_url(binaries, bases[1], 'bash'):
'SEEN BASH',
fake_url(binaries, bases[1], 'protoc'):
'UNSEEN PROTOC 1',
fake_url(binaries, bases[2], 'protoc'):
'UNSEEN PROTOC 2',
fake_url(binaries, bases[2], 'ivy'):
'UNSEEN IVY 2',
})
unseen = [
item for item in fetcher.values() if item.startswith('SEEN ')
]
for supportdir, version, name in binaries.values():
binary_path = binary_util._select_binary_base_path(
supportdir=supportdir, version=version, name=name)
with binary_util._select_binary_stream(name=name,
binary_path=binary_path,
fetcher=fetcher) as stream:
result = stream()
self.assertEqual(result, 'SEEN ' + name.upper())
unseen.remove(result)
self.assertEqual(0, len(unseen)) # Make sure we've seen all the SEENs.
def test_support_url_multi(self):
"""Tests to make sure existing base urls function as expected."""
count = 0
binary_util = BinaryUtil(
baseurls=[
'BLATANTLY INVALID URL',
'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
'https://dl.bintray.com/pantsbuild/bin/build-support',
'https://dl.bintray.com/pantsbuild/bin/build-support', # Test duplicate entry handling.
'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
],
timeout_secs=30,
bootstrapdir='/tmp')
with binary_util._select_binary_stream(supportdir='bin/protobuf',
version='2.4.1',
name='protoc') as stream:
stream()
count += 1
self.assertEqual(count, 1)
def test_support_url_multi(self):
"""Tests to make sure existing base urls function as expected."""
with temporary_dir() as invalid_local_files, temporary_dir() as valid_local_files:
binary_util = BinaryUtil(
baseurls=[
'BLATANTLY INVALID URL',
'https://dl.bintray.com/pantsbuild/bin/reasonably-invalid-url',
invalid_local_files,
valid_local_files,
'https://dl.bintray.com/pantsbuild/bin/another-invalid-url',
],
timeout_secs=30,
bootstrapdir='/tmp')
binary_path = binary_util._select_binary_base_path(supportdir='bin/protobuf',
version='2.4.1',
name='protoc')
contents = b'proof'
with safe_open(os.path.join(valid_local_files, binary_path), 'wb') as fp:
fp.write(contents)
with binary_util._select_binary_stream(name='protoc', binary_path=binary_path) as stream:
self.assertEqual(contents, stream())
def eslint_supportdir(self, task_workdir):
""" Returns the path where the ESLint is bootstrapped.
:param string task_workdir: The task's working directory
:returns: The path where ESLint is bootstrapped and whether or not it is configured
:rtype: (string, bool)
"""
bootstrapped_support_path = os.path.join(task_workdir, 'eslint')
# TODO(nsaechao): Should only have to check if the "eslint" dir exists in the task_workdir
# assuming fingerprinting works as intended.
# If the eslint_setupdir is not provided or missing required files, then
# clean up the directory so that Pants can install a pre-defined eslint version later on.
# Otherwise, if there is no configurations changes, rely on the cache.
# If there is a config change detected, use the new configuration.
configured = False
if self.eslint_setupdir:
configured = self._binary_util.is_bin_valid(
self.eslint_setupdir, [
BinaryUtil.BinaryFileSpec('package.json'),
BinaryUtil.BinaryFileSpec('yarn.lock')
])
if not configured:
safe_mkdir(bootstrapped_support_path, clean=True)
else:
binary_file_specs = [
BinaryUtil.BinaryFileSpec(
f, hash_file(os.path.join(self.eslint_setupdir, f)))
for f in ['yarn.lock', 'package.json']
]
installed = self._binary_util.is_bin_valid(
bootstrapped_support_path, binary_file_specs)
if not installed:
self._configure_eslinter(bootstrapped_support_path)
return bootstrapped_support_path, configured
def create(cls, bootstrap_options):
"""
:param Options bootstrap_options: The bootstrap options bag.
"""
binary_util = BinaryUtil(bootstrap_options.binaries_baseurls,
bootstrap_options.binaries_fetch_timeout_secs,
bootstrap_options.pants_bootstrapdir,
bootstrap_options.binaries_path_by_id)
return WatchmanLauncher(binary_util, bootstrap_options.level,
bootstrap_options.watchman_version,
bootstrap_options.watchman_supportdir,
bootstrap_options.watchman_startup_timeout,
bootstrap_options.watchman_socket_timeout,
bootstrap_options.watchman_socket_path,
bootstrap_options.pants_subprocessdir)
async def get_binary_tool_urls(
req: BinaryToolFetchRequest,
binary_util: BinaryUtil,
) -> BinaryToolUrlSet:
tool = req.tool
platform_constraint = req.platform_constraint
mapping = VersionDigestMapping(tool.get_options().version_digest_mapping)
tool_for_platform = mapping.get(platform_constraint)
version = tool_for_platform.version.version
url_generator = binary_util.get_url_generator(
tool.make_binary_request(version))
host_platform = await Get[HostPlatform](PlatformConstraint,
platform_constraint)
return BinaryToolUrlSet(
tool_for_platform=tool_for_platform,
host_platform=host_platform,
url_generator=url_generator,
)
def _fake_url(cls, binaries, base, binary_key):
binary_util = BinaryUtil([], 0, '/tmp')
supportdir, version, name = binaries[binary_key]
binary = binary_util._select_binary_base_path(supportdir, version, binary_key)
return '{base}/{binary}'.format(base=base, binary=binary)