def _request(self, method, path, body=None):
url = self._url_string(path)
logger.debug('Sending {0} request to {1}'.format(method, url))
session = RequestsSession.instance()
try:
response = None
if 'PUT' == method:
response = session.put(url, data=body, timeout=self._timeout_secs)
elif 'GET' == method:
response = session.get(url, timeout=self._timeout_secs, stream=True)
elif 'HEAD' == method:
response = session.head(url, timeout=self._timeout_secs)
elif 'DELETE' == method:
response = session.delete(url, timeout=self._timeout_secs)
else:
raise ValueError('Unknown request method {0}'.format(method))
# Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204.
if int(response.status_code / 100) == 2:
return response
elif response.status_code == 404:
logger.debug('404 returned for {0} request to {1}'.format(method, self._url_string(path)))
return None
else:
raise NonfatalArtifactCacheError('Failed to {0} {1}. Error: {2} {3}'.format(method,
self._url_string(path),
response.status_code,
response.reason))
except RequestException as e:
raise NonfatalArtifactCacheError(e)
def try_insert(self, cache_key, paths):
# Delegate creation of artifact to local cache.
with self._localcache.insert_paths(cache_key, paths) as tarfile:
# Upload local artifact to remote cache.
with open(tarfile, 'rb') as infile:
if not self._request('PUT', cache_key, body=infile):
raise NonfatalArtifactCacheError('Failed to PUT {0}.'.format(cache_key))
def try_insert(self, cache_key, paths):
# Delegate creation of artifact to local cache.
with self._localcache.insert_paths(cache_key, paths) as tarfile:
# Upload local artifact to remote cache.
with open(tarfile, 'rb') as infile:
remote_path = self._remote_path_for_key(cache_key)
if not self._request('PUT', remote_path, body=infile):
url = self._url_string(remote_path)
raise NonfatalArtifactCacheError('Failed to PUT to {0}.'.format(url))
def try_insert(self, cache_key, paths):
logger.debug('Insert {0}'.format(cache_key))
# Delegate creation of artifacts to the local cache
with self._localcache.insert_paths(cache_key, paths) as tarfile:
with open(tarfile, 'rb') as infile:
# Upload artifact to the remote cache.
try:
response = self._get_object(cache_key).put(Body=infile)
response_status = response['ResponseMetadata'][
'HTTPStatusCode']
if response_status < 200 or response_status >= 300:
raise NonfatalArtifactCacheError(
'Failed to PUT (http error) {0}: {1}'.format(
cache_key, response_status))
except Exception as e:
raise NonfatalArtifactCacheError(
'Failed to PUT (core error) {0}: {1}'.format(
cache_key, str(e)))
def _request(self, method, cache_key, body=None):
session = RequestsSession.instance()
with self.best_url_selector.select_best_url() as best_url:
url = self._url_for_key(best_url, cache_key)
logger.debug('Sending {0} request to {1}'.format(method, url))
try:
if 'PUT' == method:
response = session.put(url,
data=body,
timeout=self._write_timeout_secs,
allow_redirects=True)
elif 'GET' == method:
response = session.get(url,
timeout=self._read_timeout_secs,
stream=True,
allow_redirects=True)
elif 'HEAD' == method:
response = session.head(url,
timeout=self._read_timeout_secs,
allow_redirects=True)
elif 'DELETE' == method:
response = session.delete(url,
timeout=self._write_timeout_secs,
allow_redirects=True)
else:
raise ValueError(
'Unknown request method {0}'.format(method))
except RequestException as e:
raise NonfatalArtifactCacheError(
'Failed to {0} {1}. Error: {2}'.format(method, url, e))
# Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204.
if int(response.status_code / 100) == 2:
return response
elif response.status_code == 404:
logger.debug('404 returned for {0} request to {1}'.format(
method, url))
return None
else:
raise NonfatalArtifactCacheError(
'Failed to {0} {1}. Error: {2} {3}'.format(
method, url, response.status_code, response.reason))
def _request_session(self, method,
url) -> Generator[requests.Session, None, None]:
try:
logger.debug(f"Sending {method} request to {url}")
# TODO: fix memo.py so @memoized_classmethod is correctly recognized by mypy!
yield RequestsSession.session() # type: ignore[call-arg]
except RequestException as e:
if RequestsSession._instance().should_check_for_max_retry_error(
): # type: ignore[call-arg]
# TODO: Determine if there's a more canonical way to extract a MaxRetryError from a
# RequestException.
base_exc = e.args[0]
if isinstance(base_exc, MaxRetryError):
raise base_exc from e
raise NonfatalArtifactCacheError(
f"Failed to {method} {url}. Error: {e}") from e
def _request(self,
method,
cache_key,
body=None) -> Optional[requests.Response]:
# If our connection pool has experienced too many retries, we no-op on every successive
# artifact download for the rest of the pants process lifetime.
if RequestsSession.has_exceeded_retries():
return None
with self.best_url_selector.select_best_url() as best_url:
url = self._url_for_key(best_url, cache_key)
with self._request_session(method, url) as session:
if "PUT" == method:
response = session.put(url,
data=body,
timeout=self._write_timeout_secs,
allow_redirects=True)
elif "GET" == method:
response = session.get(url,
timeout=self._read_timeout_secs,
stream=True,
allow_redirects=True)
elif "HEAD" == method:
response = session.head(url,
timeout=self._read_timeout_secs,
allow_redirects=True)
elif "DELETE" == method:
response = session.delete(url,
timeout=self._write_timeout_secs,
allow_redirects=True)
else:
raise ValueError(
"Unknown request method {0}".format(method))
# Allow all 2XX responses. E.g., nginx returns 201 on PUT. HEAD may return 204.
if int(response.status_code / 100) == 2:
return response
elif response.status_code == 404:
logger.debug("404 returned for {0} request to {1}".format(
method, url))
return None
else:
raise NonfatalArtifactCacheError(
"Failed to {0} {1}. Error: {2} {3}".format(
method, url, response.status_code, response.reason))
class ArtifactCacheStatsTest(TestBase):
TEST_CACHE_NAME_1 = 'ZincCompile'
TEST_CACHE_NAME_2 = 'Checkstyle_test_checkstyle'
TEST_LOCAL_ERROR = UnreadableArtifact('foo',
ArtifactError('CRC check failed'))
TEST_REMOTE_ERROR = UnreadableArtifact(
'bar',
NonfatalArtifactCacheError(
requests.exceptions.ConnectionError('Read time out')))
TEST_SPEC_A = 'src/scala/a'
TEST_SPEC_B = 'src/scala/b'
TEST_SPEC_C = 'src/java/c'
def setUp(self):
super().setUp()
self.target_a = self.make_target(spec=self.TEST_SPEC_A)
self.target_b = self.make_target(spec=self.TEST_SPEC_B)
self.target_c = self.make_target(spec=self.TEST_SPEC_C)
def test_add_hits(self):
expected_stats = [
{
'cache_name': self.TEST_CACHE_NAME_2,
'num_hits': 0,
'num_misses': 1,
'hits': [],
'misses': [(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err))]
},
{
'cache_name': self.TEST_CACHE_NAME_1,
'num_hits': 1,
'num_misses': 1,
'hits': [(self.TEST_SPEC_B, '')],
'misses': [(self.TEST_SPEC_C, str(self.TEST_REMOTE_ERROR.err))]
},
]
expected_hit_or_miss_files = {
'{}.misses'.format(self.TEST_CACHE_NAME_2):
'{} {}\n'.format(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err)),
'{}.hits'.format(self.TEST_CACHE_NAME_1):
'{}\n'.format(self.TEST_SPEC_B),
'{}.misses'.format(self.TEST_CACHE_NAME_1):
'{} {}\n'.format(self.TEST_SPEC_C,
str(self.TEST_REMOTE_ERROR.err)),
}
with self.mock_artifact_cache_stats(expected_stats,
expected_hit_or_miss_files=expected_hit_or_miss_files)\
as artifact_cache_stats:
artifact_cache_stats.add_hits(self.TEST_CACHE_NAME_1,
[self.target_b])
artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_1,
[self.target_c],
[self.TEST_REMOTE_ERROR])
artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_2,
[self.target_a],
[self.TEST_LOCAL_ERROR])
@contextmanager
def mock_artifact_cache_stats(self,
expected_stats,
expected_hit_or_miss_files=None):
with temporary_dir() as tmp_dir:
artifact_cache_stats = ArtifactCacheStats(tmp_dir)
yield artifact_cache_stats
self.assertEqual(
sorted(expected_stats, key=lambda s: s['cache_name']),
sorted(artifact_cache_stats.get_all(),
key=lambda s: s['cache_name']))
self.assertEqual(sorted(list(expected_hit_or_miss_files.keys())),
sorted(os.listdir(tmp_dir)))
for hit_or_miss_file in expected_hit_or_miss_files.keys():
with open(os.path.join(tmp_dir, hit_or_miss_file),
'r') as hit_or_miss_saved:
self.assertEqual(
expected_hit_or_miss_files[hit_or_miss_file],
hit_or_miss_saved.read())
class ArtifactCacheStatsTest(TestBase):
TEST_CACHE_NAME_1 = "ZincCompile"
TEST_CACHE_NAME_2 = "Checkstyle_test_checkstyle"
TEST_LOCAL_ERROR = UnreadableArtifact("foo",
ArtifactError("CRC check failed"))
TEST_REMOTE_ERROR = UnreadableArtifact(
"bar",
NonfatalArtifactCacheError(
requests.exceptions.ConnectionError("Read time out")))
TEST_SPEC_A = "src/scala/a"
TEST_SPEC_B = "src/scala/b"
TEST_SPEC_C = "src/java/c"
def setUp(self):
super().setUp()
self.target_a = self.make_target(spec=self.TEST_SPEC_A)
self.target_b = self.make_target(spec=self.TEST_SPEC_B)
self.target_c = self.make_target(spec=self.TEST_SPEC_C)
def test_add_hits(self):
expected_stats = [
{
"cache_name": self.TEST_CACHE_NAME_2,
"num_hits": 0,
"num_misses": 1,
"hits": [],
"misses": [(self.TEST_SPEC_A, str(self.TEST_LOCAL_ERROR.err))],
},
{
"cache_name": self.TEST_CACHE_NAME_1,
"num_hits": 1,
"num_misses": 1,
"hits": [(self.TEST_SPEC_B, "")],
"misses":
[(self.TEST_SPEC_C, str(self.TEST_REMOTE_ERROR.err))],
},
]
expected_hit_or_miss_files = {
f"{self.TEST_CACHE_NAME_2}.misses":
f"{self.TEST_SPEC_A} {str(self.TEST_LOCAL_ERROR.err)}\n",
f"{self.TEST_CACHE_NAME_1}.hits":
f"{self.TEST_SPEC_B}\n",
f"{self.TEST_CACHE_NAME_1}.misses":
f"{self.TEST_SPEC_C} {str(self.TEST_REMOTE_ERROR.err)}\n",
}
with self.mock_artifact_cache_stats(
expected_stats,
expected_hit_or_miss_files=expected_hit_or_miss_files
) as artifact_cache_stats:
artifact_cache_stats.add_hits(self.TEST_CACHE_NAME_1,
[self.target_b])
artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_1,
[self.target_c],
[self.TEST_REMOTE_ERROR])
artifact_cache_stats.add_misses(self.TEST_CACHE_NAME_2,
[self.target_a],
[self.TEST_LOCAL_ERROR])
@contextmanager
def mock_artifact_cache_stats(self,
expected_stats,
expected_hit_or_miss_files=None):
with temporary_dir() as tmp_dir:
artifact_cache_stats = ArtifactCacheStats(tmp_dir)
yield artifact_cache_stats
self.assertEqual(
sorted(expected_stats, key=lambda s: s["cache_name"]),
sorted(artifact_cache_stats.get_all(),
key=lambda s: s["cache_name"]),
)
self.assertEqual(sorted(list(expected_hit_or_miss_files.keys())),
sorted(os.listdir(tmp_dir)))
for hit_or_miss_file in expected_hit_or_miss_files.keys():
with open(os.path.join(tmp_dir, hit_or_miss_file),
"r") as hit_or_miss_saved:
self.assertEqual(
expected_hit_or_miss_files[hit_or_miss_file],
hit_or_miss_saved.read())
