def reader_release(self):
with self._lock:
ASSERT.greater(self._num_readers, 0)
ASSERT.equal(self._num_writers, 0)
self._num_readers -= 1
if self._num_readers == 0:
self._writer_cond.notify()
def __init__(self, token_rate, bucket_size, raise_when_empty):
self._raise_when_empty = raise_when_empty
self._token_rate = ASSERT.greater(token_rate, 0)
self._token_period = 1 / self._token_rate
self._bucket_size = ASSERT.greater(bucket_size, 0)
self._num_tokens = 0
self._last_added = time.monotonic()
def lease_dissociate(tables, *, lease, key_id):
ASSERT.greater(lease, 0)
return (
tables.leases_key_ids.delete()\
.where(tables.leases_key_ids.c.lease == lease)
.where(tables.leases_key_ids.c.key_id == key_id)
)
def lease_grant(tables, *, lease, expiration):
ASSERT.greater(lease, 0)
ASSERT.greater_or_equal(expiration, 0)
return sqlite.upsert(tables.leases).values(
lease=lease,
expiration=expiration,
)
def __init__(
self,
*,
num_buckets=512,
token_rate,
bucket_size,
get_bucket_key=default_get_bucket_key,
get_num_needed=lambda _: 1,
):
self._buckets = g1_collections.LruCache(num_buckets)
self._get_bucket_key = get_bucket_key
self._get_num_needed = get_num_needed
self._token_rate = ASSERT.greater(token_rate, 0)
self._bucket_size = ASSERT.greater(bucket_size, 0)
def __init__(
self,
*,
key,
failure_threshold,
failure_period,
failure_timeout,
success_threshold,
):
self._key = key
self._failure_threshold = ASSERT.greater(failure_threshold, 0)
self._failure_period = ASSERT.greater(failure_period, 0)
self._failure_timeout = ASSERT.greater(failure_timeout, 0)
self._success_threshold = ASSERT.greater(success_threshold, 0)
self._state = _States.GREEN
# When state is GREEN, _event_log records failures; when state
# is YELLOW, it records successes; when state is RED, it records
# when the state was changed to RED.
self._event_log = _EventLog(max(failure_threshold, success_threshold))
self._num_concurrent_requests = 0
def __init__(
self,
cache_dir_path,
capacity,
*,
post_eviction_size=None,
executor=None, # Use this to evict in the background.
):
self._lock = threading.Lock()
self._cache_dir_path = ASSERT.predicate(cache_dir_path, Path.is_dir)
self._capacity = ASSERT.greater(capacity, 0)
self._post_eviction_size = (post_eviction_size
if post_eviction_size is not None else
int(self._capacity *
POST_EVICTION_SIZE_RATIO))
ASSERT(
0 <= self._post_eviction_size <= self._capacity,
'expect 0 <= post_eviction_size <= {}, not {}',
self._capacity,
self._post_eviction_size,
)
self._executor = executor
# By the way, if cache cold start is an issue, we could store
# and load this table from a file.
self._access_log = collections.OrderedDict()
# getting_path may "lease" paths to the user, and we should not
# evict these paths.
self._active_paths = g1_collections.Multiset()
self._num_hits = 0
self._num_misses = 0
# It's safe to call these methods after this point.
self._eviction_countdown = self._estimate_eviction_countdown()
self._maybe_evict()
async def recvfile(response, file):
"""Receive response body into a file.
The caller must set ``stream`` to true when make the request.
DANGER! This breaks the multiple levels of encapsulation, from
requests.Response all the way down to http.client.HTTPResponse.
As a result, the response object is most likely unusable after a
recvfile call, and you should probably close it immediately.
"""
# requests sets _content to False initially.
ASSERT.is_(response._content, False)
ASSERT.false(response._content_consumed)
urllib3_response = ASSERT.not_none(response.raw)
chunked = urllib3_response.chunked
httplib_response = ASSERT.isinstance(
urllib3_response._fp, http.client.HTTPResponse
)
ASSERT.false(httplib_response.closed)
sock = ASSERT.isinstance(httplib_response.fp.raw._sock, socket.socket)
output = DecoderChain(file)
if chunked:
chunk_decoder = ChunkDecoder()
output.add(chunk_decoder)
num_to_read = 0
eof = lambda: chunk_decoder.eof
else:
num_to_read = ASSERT.greater(
ASSERT.not_none(httplib_response.length), 0
)
eof = lambda: num_to_read <= 0
# Use urllib3's decoder code.
urllib3_response._init_decoder()
if urllib3_response._decoder is not None:
output.add(ContentDecoder(urllib3_response._decoder))
with contextlib.ExitStack() as stack:
src = adapters.FileAdapter(httplib_response.fp)
stack.callback(src.disown)
sock.setblocking(False)
stack.callback(sock.setblocking, True)
buffer = memoryview(stack.enter_context(_BUFFER_POOL.using()))
while not eof():
if chunked:
# TODO: If server sends more data at the end, like
# response of the next request, for now recvfile might
# read them, and then err out. Maybe recvfile should
# check this, and not read more than it should instead?
num_read = await src.readinto1(buffer)
else:
num_read = await src.readinto1(
buffer[:min(num_to_read, _CHUNK_SIZE)]
)
if num_read == 0:
break
output.write(buffer[:num_read])
num_to_read -= num_read
output.flush()
# Sanity check.
if not chunked:
ASSERT.equal(num_to_read, 0)
# Trick requests to release the connection back to the connection
# pool, rather than closing/discarding it.
response._content_consumed = True
# http.client.HTTPConnection tracks the last response; so you have
# to close it to make the connection object useable again.
httplib_response.close()
# Close the response for the caller since response is not useable
# after recvfile.
response.close()
loggings.ONCE_PER(
1000, LOG.info, 'buffer pool stats: %r', _BUFFER_POOL.get_stats()
)
def add(self, t):
if self._log:
ASSERT.greater(t, self._log[-1])
self._log.append(t)
def __init__(self, max_retries, backoff_base):
self._max_retries = ASSERT.greater(max_retries, 0)
self._backoff_base = ASSERT.greater(backoff_base, 0)
def __init__(self, token_rate, bucket_size):
self._token_rate = ASSERT.greater(token_rate, 0)
self._bucket_size = ASSERT.greater(bucket_size, 0)
self._num_tokens = self._bucket_size # Bucket is full initially.
self._last_added = time.monotonic()
def lease_revoke(tables, *, lease):
ASSERT.greater(lease, 0)
return [
table.delete().where(table.c.lease == lease)
for table in (tables.leases, tables.leases_key_ids)
]
def lease_associate(tables, *, lease, key_id):
ASSERT.greater(lease, 0)
return sqlite.upsert(tables.leases_key_ids).values(
lease=lease,
key_id=key_id,
)
def __post_init__(self):
ASSERT.greater(self.revision, 0)
ASSERT.true(self.key)
def __post_init__(self):
ASSERT.greater(self.lease, 0)
ASSERT.greater_or_equal(self.expiration, 0)
