def test_checksum_listener(self): digest = mock.Mock(spec=hashlib.md5()) digest.hexdigest.return_value = '42' checksum_listener = Fetcher.ChecksumListener(digest=digest) with self.expect_get('http://baz', chunk_size_bytes=1, timeout_secs=37) as (chunks, expected_listener_calls): self.fetcher.fetch('http://baz', checksum_listener.wrap(self.listener), chunk_size_bytes=1, timeout_secs=37) self.assertEqual('42', checksum_listener.checksum) def expected_digest_calls(): for chunk in chunks: yield mock.call.update(chunk) yield mock.call.hexdigest() self.assertEqual(list(expected_digest_calls()), digest.method_calls) self.assert_listener_calls(expected_listener_calls, chunks) self.response.close.assert_called_once_with()
def _bootstrap_ivy(self, bootstrap_jar_path): options = self._ivy_subsystem.get_options() if not os.path.exists(bootstrap_jar_path): with temporary_file() as bootstrap_jar: fetcher = Fetcher(get_buildroot()) checksummer = fetcher.ChecksumListener(digest=hashlib.sha1()) try: logger.info('\nDownloading {}'.format( options.bootstrap_jar_url)) # TODO: Capture the stdout of the fetcher, instead of letting it output # to the console directly. fetcher.download( options.bootstrap_jar_url, listener=fetcher.ProgressListener().wrap(checksummer), path_or_fd=bootstrap_jar, timeout_secs=options.bootstrap_fetch_timeout_secs) logger.info('sha1: {}'.format(checksummer.checksum)) bootstrap_jar.close() touch(bootstrap_jar_path) shutil.move(bootstrap_jar.name, bootstrap_jar_path) except fetcher.Error as e: raise self.Error( 'Problem fetching the ivy bootstrap jar! {}'.format(e)) return Ivy(bootstrap_jar_path, ivy_settings=options.bootstrap_ivy_settings or options.ivy_settings, ivy_resolution_cache_dir=self._ivy_subsystem. resolution_cache_dir(), extra_jvm_options=self._ivy_subsystem.extra_jvm_options())
def _bootstrap_ivy(self, bootstrap_jar_path): if not os.path.exists(bootstrap_jar_path): with temporary_file() as bootstrap_jar: fetcher = Fetcher() checksummer = fetcher.ChecksumListener(digest=hashlib.sha1()) try: log.info('\nDownloading %s' % self._bootstrap_jar_url) # TODO: Capture the stdout of the fetcher, instead of letting it output # to the console directly. fetcher.download( self._bootstrap_jar_url, listener=fetcher.ProgressListener().wrap(checksummer), path_or_fd=bootstrap_jar, timeout_secs=self._timeout_secs) log.info('sha1: %s' % checksummer.checksum) bootstrap_jar.close() touch(bootstrap_jar_path) shutil.move(bootstrap_jar.name, bootstrap_jar_path) except fetcher.Error as e: raise self.Error( 'Problem fetching the ivy bootstrap jar! %s' % e) return Ivy(bootstrap_jar_path, ivy_settings=self._ivy_settings, ivy_cache_dir=self.ivy_cache_dir)
def bootstrap_coursier(self, workunit_factory): opts = self.get_options() bootstrap_url = opts.bootstrap_jar_url coursier_bootstrap_dir = os.path.join(opts.pants_bootstrapdir, 'tools', 'jvm', 'coursier', opts.version) bootstrap_jar_path = os.path.join(coursier_bootstrap_dir, 'coursier.jar') if not os.path.exists(bootstrap_jar_path): with workunit_factory(name='bootstrap-coursier', labels=[WorkUnitLabel.TOOL]) as workunit: with safe_concurrent_creation(bootstrap_jar_path) as temp_path: fetcher = Fetcher(get_buildroot()) checksummer = fetcher.ChecksumListener(digest=hashlib.sha1()) try: logger.info('\nDownloading {}'.format(bootstrap_url)) # TODO: Capture the stdout of the fetcher, instead of letting it output # to the console directly. fetcher.download(bootstrap_url, listener=fetcher.ProgressListener().wrap(checksummer), path_or_fd=temp_path, timeout_secs=opts.bootstrap_fetch_timeout_secs) logger.info('sha1: {}'.format(checksummer.checksum)) except fetcher.Error as e: workunit.set_outcome(WorkUnit.FAILURE) raise self.Error('Problem fetching the coursier bootstrap jar! {}'.format(e)) else: workunit.set_outcome(WorkUnit.SUCCESS) return bootstrap_jar_path
def test_checksum_listener(self): digest = self.mox.CreateMockAnything() for chunk in self.expect_get('http://baz', chunk_size_bytes=1, timeout_secs=37): self.listener.recv_chunk(chunk) digest.update(chunk) self.listener.finished() digest.hexdigest().AndReturn('42') self.response.close() self.mox.ReplayAll() checksum_listener = Fetcher.ChecksumListener(digest=digest) self.fetcher.fetch('http://baz', checksum_listener.wrap(self.listener), chunk_size_bytes=1, timeout_secs=37) self.assertEqual('42', checksum_listener.checksum)
def fetch_prebuilt_wheels(self, binary_base_url, deploy_pants_wheels_path, deploy_3rdparty_wheels_path, to_dir): wheel_paths = self.list_prebuilt_wheels(binary_base_url, deploy_pants_wheels_path, deploy_3rdparty_wheels_path) if not wheel_paths: raise ValueError("No wheels found.") # Fetching the wheels in parallel # It is okay to have some interleaving outputs from the fetcher, # because we are summarizing things in the end. fetcher = Fetcher(os.getcwd()) checksummer = fetcher.ChecksumListener(digest=hashlib.sha1()) futures = [] with ThreadPoolExecutor(max_workers=8) as executor: for k in wheel_paths: file_path, url_path = k.split(self.OUTPUT_DELIMITER) dest = os.path.join(to_dir, file_path) safe_mkdir(os.path.dirname(dest)) url = '{}/{}'.format(binary_base_url, url_path) future = executor.submit(self._download, fetcher, checksummer, url, dest) futures.append((future, url)) # Summarize the fetch results. fail = False for future, url in futures: if future.exception() is not None: logger.error('Failed to download: {}'.format(url)) fail = True else: logger.info('Downloaded: {}'.format(url)) if fail: raise fetcher.Error()