def _request(self, method, path, body=None): url = self._url_string(path) logger.debug('Sending {0} request to {1}'.format(method, url)) session = RequestsSession.instance() try: response = None if 'PUT' == method: response = session.put(url, data=body, timeout=self._timeout_secs) elif 'GET' == method: response = session.get(url, timeout=self._timeout_secs, stream=True) elif 'HEAD' == method: response = session.head(url, timeout=self._timeout_secs) elif 'DELETE' == method: response = session.delete(url, timeout=self._timeout_secs) else: raise ValueError('Unknown request method {0}'.format(method)) # Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204. if int(response.status_code / 100) == 2: return response elif response.status_code == 404: logger.debug('404 returned for {0} request to {1}'.format(method, self._url_string(path))) return None else: raise NonfatalArtifactCacheError('Failed to {0} {1}. Error: {2} {3}'.format(method, self._url_string(path), response.status_code, response.reason)) except RequestException as e: raise NonfatalArtifactCacheError(e)
def try_insert(self, cache_key, paths): # Delegate creation of artifact to local cache. with self._localcache.insert_paths(cache_key, paths) as tarfile: # Upload local artifact to remote cache. with open(tarfile, 'rb') as infile: if not self._request('PUT', cache_key, body=infile): raise NonfatalArtifactCacheError('Failed to PUT {0}.'.format(cache_key))
def try_insert(self, cache_key, paths): # Delegate creation of artifact to local cache. with self._localcache.insert_paths(cache_key, paths) as tarfile: # Upload local artifact to remote cache. with open(tarfile, 'rb') as infile: remote_path = self._remote_path_for_key(cache_key) if not self._request('PUT', remote_path, body=infile): url = self._url_string(remote_path) raise NonfatalArtifactCacheError('Failed to PUT to {0}.'.format(url))
def try_insert(self, cache_key, paths): logger.debug('Insert {0}'.format(cache_key)) # Delegate creation of artifacts to the local cache with self._localcache.insert_paths(cache_key, paths) as tarfile: with open(tarfile, 'rb') as infile: # Upload artifact to the remote cache. try: response = self._get_object(cache_key).put(Body=infile) response_status = response['ResponseMetadata'][ 'HTTPStatusCode'] if response_status < 200 or response_status >= 300: raise NonfatalArtifactCacheError( 'Failed to PUT (http error) {0}: {1}'.format( cache_key, response_status)) except Exception as e: raise NonfatalArtifactCacheError( 'Failed to PUT (core error) {0}: {1}'.format( cache_key, str(e)))
def _request(self, method, cache_key, body=None): session = RequestsSession.instance() with self.best_url_selector.select_best_url() as best_url: url = self._url_for_key(best_url, cache_key) logger.debug('Sending {0} request to {1}'.format(method, url)) try: if 'PUT' == method: response = session.put(url, data=body, timeout=self._write_timeout_secs, allow_redirects=True) elif 'GET' == method: response = session.get(url, timeout=self._read_timeout_secs, stream=True, allow_redirects=True) elif 'HEAD' == method: response = session.head(url, timeout=self._read_timeout_secs, allow_redirects=True) elif 'DELETE' == method: response = session.delete(url, timeout=self._write_timeout_secs, allow_redirects=True) else: raise ValueError( 'Unknown request method {0}'.format(method)) except RequestException as e: raise NonfatalArtifactCacheError( 'Failed to {0} {1}. Error: {2}'.format(method, url, e)) # Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204. if int(response.status_code / 100) == 2: return response elif response.status_code == 404: logger.debug('404 returned for {0} request to {1}'.format( method, url)) return None else: raise NonfatalArtifactCacheError( 'Failed to {0} {1}. Error: {2} {3}'.format( method, url, response.status_code, response.reason))
def _request_session(self, method, url) -> Generator[requests.Session, None, None]: try: logger.debug(f"Sending {method} request to {url}") # TODO: fix memo.py so @memoized_classmethod is correctly recognized by mypy! yield RequestsSession.session() # type: ignore[call-arg] except RequestException as e: if RequestsSession._instance().should_check_for_max_retry_error( ): # type: ignore[call-arg] # TODO: Determine if there's a more canonical way to extract a MaxRetryError from a # RequestException. base_exc = e.args[0] if isinstance(base_exc, MaxRetryError): raise base_exc from e raise NonfatalArtifactCacheError( f"Failed to {method} {url}. Error: {e}") from e
def _request(self, method, cache_key, body=None) -> Optional[requests.Response]: # If our connection pool has experienced too many retries, we no-op on every successive # artifact download for the rest of the pants process lifetime. if RequestsSession.has_exceeded_retries(): return None with self.best_url_selector.select_best_url() as best_url: url = self._url_for_key(best_url, cache_key) with self._request_session(method, url) as session: if "PUT" == method: response = session.put(url, data=body, timeout=self._write_timeout_secs, allow_redirects=True) elif "GET" == method: response = session.get(url, timeout=self._read_timeout_secs, stream=True, allow_redirects=True) elif "HEAD" == method: response = session.head(url, timeout=self._read_timeout_secs, allow_redirects=True) elif "DELETE" == method: response = session.delete(url, timeout=self._write_timeout_secs, allow_redirects=True) else: raise ValueError( "Unknown request method {0}".format(method)) # Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204. if int(response.status_code / 100) == 2: return response elif response.status_code == 404: logger.debug("404 returned for {0} request to {1}".format( method, url)) return None else: raise NonfatalArtifactCacheError( "Failed to {0} {1}. Error: {2} {3}".format( method, url, response.status_code, response.reason))
class ArtifactCacheStatsTest(TestBase): TEST_CACHE_NAME_1 = 'ZincCompile' TEST_CACHE_NAME_2 = 'Checkstyle_test_checkstyle' TEST_LOCAL_ERROR = UnreadableArtifact('foo', ArtifactError('CRC check failed')) TEST_REMOTE_ERROR = UnreadableArtifact( 'bar', NonfatalArtifactCacheError( requests.exceptions.ConnectionError('Read time out'))) TEST_SPEC_A = 'src/scala/a' TEST_SPEC_B = 'src/scala/b' TEST_SPEC_C = 'src/java/c' def setUp(self): super().setUp() self.target_a = self.make_target(spec=self.TEST_SPEC_A) self.target_b = self.make_target(spec=self.TEST_SPEC_B) self.target_c = self.make_target(spec=self.TEST_SPEC_C) def test_add_hits(self): expected_stats = [ { 'cache_name': self.TEST_CACHE_NAME_2, 'num_hits': 0, 'num_misses': 1, 'hits': [], 'misses': [(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err))] }, { 'cache_name': self.TEST_CACHE_NAME_1, 'num_hits': 1, 'num_misses': 1, 'hits': [(self.TEST_SPEC_B, '')], 'misses': [(self.TEST_SPEC_C, str(self.TEST_REMOTE_ERROR.err))] }, ] expected_hit_or_miss_files = { '{}.misses'.format(self.TEST_CACHE_NAME_2): '{} {}\n'.format(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err)), '{}.hits'.format(self.TEST_CACHE_NAME_1): '{}\n'.format(self.TEST_SPEC_B), '{}.misses'.format(self.TEST_CACHE_NAME_1): '{} {}\n'.format(self.TEST_SPEC_C, str(self.TEST_REMOTE_ERROR.err)), } with self.mock_artifact_cache_stats(expected_stats, expected_hit_or_miss_files=expected_hit_or_miss_files)\ as artifact_cache_stats: artifact_cache_stats.add_hits(self.TEST_CACHE_NAME_1, [self.target_b]) artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_1, [self.target_c], [self.TEST_REMOTE_ERROR]) artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_2, [self.target_a], [self.TEST_LOCAL_ERROR]) @contextmanager def mock_artifact_cache_stats(self, expected_stats, expected_hit_or_miss_files=None): with temporary_dir() as tmp_dir: artifact_cache_stats = ArtifactCacheStats(tmp_dir) yield artifact_cache_stats self.assertEqual( sorted(expected_stats, key=lambda s: s['cache_name']), sorted(artifact_cache_stats.get_all(), key=lambda s: s['cache_name'])) self.assertEqual(sorted(list(expected_hit_or_miss_files.keys())), sorted(os.listdir(tmp_dir))) for hit_or_miss_file in expected_hit_or_miss_files.keys(): with open(os.path.join(tmp_dir, hit_or_miss_file), 'r') as hit_or_miss_saved: self.assertEqual( expected_hit_or_miss_files[hit_or_miss_file], hit_or_miss_saved.read())
class ArtifactCacheStatsTest(TestBase): TEST_CACHE_NAME_1 = "ZincCompile" TEST_CACHE_NAME_2 = "Checkstyle_test_checkstyle" TEST_LOCAL_ERROR = UnreadableArtifact("foo", ArtifactError("CRC check failed")) TEST_REMOTE_ERROR = UnreadableArtifact( "bar", NonfatalArtifactCacheError( requests.exceptions.ConnectionError("Read time out"))) TEST_SPEC_A = "src/scala/a" TEST_SPEC_B = "src/scala/b" TEST_SPEC_C = "src/java/c" def setUp(self): super().setUp() self.target_a = self.make_target(spec=self.TEST_SPEC_A) self.target_b = self.make_target(spec=self.TEST_SPEC_B) self.target_c = self.make_target(spec=self.TEST_SPEC_C) def test_add_hits(self): expected_stats = [ { "cache_name": self.TEST_CACHE_NAME_2, "num_hits": 0, "num_misses": 1, "hits": [], "misses": [(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err))], }, { "cache_name": self.TEST_CACHE_NAME_1, "num_hits": 1, "num_misses": 1, "hits": [(self.TEST_SPEC_B, "")], "misses": [(self.TEST_SPEC_C, str(self.TEST_REMOTE_ERROR.err))], }, ] expected_hit_or_miss_files = { f"{self.TEST_CACHE_NAME_2}.misses": f"{self.TEST_SPEC_A} {str(self.TEST_LOCAL_ERROR.err)}\n", f"{self.TEST_CACHE_NAME_1}.hits": f"{self.TEST_SPEC_B}\n", f"{self.TEST_CACHE_NAME_1}.misses": f"{self.TEST_SPEC_C} {str(self.TEST_REMOTE_ERROR.err)}\n", } with self.mock_artifact_cache_stats( expected_stats, expected_hit_or_miss_files=expected_hit_or_miss_files ) as artifact_cache_stats: artifact_cache_stats.add_hits(self.TEST_CACHE_NAME_1, [self.target_b]) artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_1, [self.target_c], [self.TEST_REMOTE_ERROR]) artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_2, [self.target_a], [self.TEST_LOCAL_ERROR]) @contextmanager def mock_artifact_cache_stats(self, expected_stats, expected_hit_or_miss_files=None): with temporary_dir() as tmp_dir: artifact_cache_stats = ArtifactCacheStats(tmp_dir) yield artifact_cache_stats self.assertEqual( sorted(expected_stats, key=lambda s: s["cache_name"]), sorted(artifact_cache_stats.get_all(), key=lambda s: s["cache_name"]), ) self.assertEqual(sorted(list(expected_hit_or_miss_files.keys())), sorted(os.listdir(tmp_dir))) for hit_or_miss_file in expected_hit_or_miss_files.keys(): with open(os.path.join(tmp_dir, hit_or_miss_file), "r") as hit_or_miss_saved: self.assertEqual( expected_hit_or_miss_files[hit_or_miss_file], hit_or_miss_saved.read())