def test_checksum_listener(self):
digest = mock.Mock(spec=hashlib.md5())
digest.hexdigest.return_value = '42'
checksum_listener = Fetcher.ChecksumListener(digest=digest)
with self.expect_get('http://baz', chunk_size_bytes=1,
timeout_secs=37) as (chunks,
expected_listener_calls):
self.fetcher.fetch('http://baz',
checksum_listener.wrap(self.listener),
chunk_size_bytes=1,
timeout_secs=37)
self.assertEqual('42', checksum_listener.checksum)
def expected_digest_calls():
for chunk in chunks:
yield mock.call.update(chunk)
yield mock.call.hexdigest()
self.assertEqual(list(expected_digest_calls()), digest.method_calls)
self.assert_listener_calls(expected_listener_calls, chunks)
self.response.close.assert_called_once_with()
def _bootstrap_ivy(self, bootstrap_jar_path):
options = self._ivy_subsystem.get_options()
if not os.path.exists(bootstrap_jar_path):
with temporary_file() as bootstrap_jar:
fetcher = Fetcher(get_buildroot())
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
try:
logger.info('\nDownloading {}'.format(
options.bootstrap_jar_url))
# TODO: Capture the stdout of the fetcher, instead of letting it output
# to the console directly.
fetcher.download(
options.bootstrap_jar_url,
listener=fetcher.ProgressListener().wrap(checksummer),
path_or_fd=bootstrap_jar,
timeout_secs=options.bootstrap_fetch_timeout_secs)
logger.info('sha1: {}'.format(checksummer.checksum))
bootstrap_jar.close()
touch(bootstrap_jar_path)
shutil.move(bootstrap_jar.name, bootstrap_jar_path)
except fetcher.Error as e:
raise self.Error(
'Problem fetching the ivy bootstrap jar! {}'.format(e))
return Ivy(bootstrap_jar_path,
ivy_settings=options.bootstrap_ivy_settings
or options.ivy_settings,
ivy_resolution_cache_dir=self._ivy_subsystem.
resolution_cache_dir(),
extra_jvm_options=self._ivy_subsystem.extra_jvm_options())
def _bootstrap_ivy(self, bootstrap_jar_path):
if not os.path.exists(bootstrap_jar_path):
with temporary_file() as bootstrap_jar:
fetcher = Fetcher()
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
try:
log.info('\nDownloading %s' % self._bootstrap_jar_url)
# TODO: Capture the stdout of the fetcher, instead of letting it output
# to the console directly.
fetcher.download(
self._bootstrap_jar_url,
listener=fetcher.ProgressListener().wrap(checksummer),
path_or_fd=bootstrap_jar,
timeout_secs=self._timeout_secs)
log.info('sha1: %s' % checksummer.checksum)
bootstrap_jar.close()
touch(bootstrap_jar_path)
shutil.move(bootstrap_jar.name, bootstrap_jar_path)
except fetcher.Error as e:
raise self.Error(
'Problem fetching the ivy bootstrap jar! %s' % e)
return Ivy(bootstrap_jar_path,
ivy_settings=self._ivy_settings,
ivy_cache_dir=self.ivy_cache_dir)
def bootstrap_coursier(self, workunit_factory):
opts = self.get_options()
bootstrap_url = opts.bootstrap_jar_url
coursier_bootstrap_dir = os.path.join(opts.pants_bootstrapdir,
'tools', 'jvm', 'coursier',
opts.version)
bootstrap_jar_path = os.path.join(coursier_bootstrap_dir, 'coursier.jar')
if not os.path.exists(bootstrap_jar_path):
with workunit_factory(name='bootstrap-coursier', labels=[WorkUnitLabel.TOOL]) as workunit:
with safe_concurrent_creation(bootstrap_jar_path) as temp_path:
fetcher = Fetcher(get_buildroot())
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
try:
logger.info('\nDownloading {}'.format(bootstrap_url))
# TODO: Capture the stdout of the fetcher, instead of letting it output
# to the console directly.
fetcher.download(bootstrap_url,
listener=fetcher.ProgressListener().wrap(checksummer),
path_or_fd=temp_path,
timeout_secs=opts.bootstrap_fetch_timeout_secs)
logger.info('sha1: {}'.format(checksummer.checksum))
except fetcher.Error as e:
workunit.set_outcome(WorkUnit.FAILURE)
raise self.Error('Problem fetching the coursier bootstrap jar! {}'.format(e))
else:
workunit.set_outcome(WorkUnit.SUCCESS)
return bootstrap_jar_path
def test_checksum_listener(self):
digest = self.mox.CreateMockAnything()
for chunk in self.expect_get('http://baz', chunk_size_bytes=1, timeout_secs=37):
self.listener.recv_chunk(chunk)
digest.update(chunk)
self.listener.finished()
digest.hexdigest().AndReturn('42')
self.response.close()
self.mox.ReplayAll()
checksum_listener = Fetcher.ChecksumListener(digest=digest)
self.fetcher.fetch('http://baz',
checksum_listener.wrap(self.listener),
chunk_size_bytes=1,
timeout_secs=37)
self.assertEqual('42', checksum_listener.checksum)
def fetch_prebuilt_wheels(self, binary_base_url, deploy_pants_wheels_path,
deploy_3rdparty_wheels_path, to_dir):
wheel_paths = self.list_prebuilt_wheels(binary_base_url,
deploy_pants_wheels_path,
deploy_3rdparty_wheels_path)
if not wheel_paths:
raise ValueError("No wheels found.")
# Fetching the wheels in parallel
# It is okay to have some interleaving outputs from the fetcher,
# because we are summarizing things in the end.
fetcher = Fetcher(os.getcwd())
checksummer = fetcher.ChecksumListener(digest=hashlib.sha1())
futures = []
with ThreadPoolExecutor(max_workers=8) as executor:
for k in wheel_paths:
file_path, url_path = k.split(self.OUTPUT_DELIMITER)
dest = os.path.join(to_dir, file_path)
safe_mkdir(os.path.dirname(dest))
url = '{}/{}'.format(binary_base_url, url_path)
future = executor.submit(self._download, fetcher, checksummer,
url, dest)
futures.append((future, url))
# Summarize the fetch results.
fail = False
for future, url in futures:
if future.exception() is not None:
logger.error('Failed to download: {}'.format(url))
fail = True
else:
logger.info('Downloaded: {}'.format(url))
if fail:
raise fetcher.Error()
