def flush(self):
with self.lock:
if self.flushed:
raise ValueError("Repeated call to flush()")
self.flushed = 1
result = self._compress(b'', action=m.LZMA_FINISH)
__pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE)
_release_lzma_stream(self.lzs)
return result
def connect(database, timeout=5.0, detect_types=0, isolation_level="",
check_same_thread=True, factory=None, cached_statements=100,
uri=0):
factory = Connection if not factory else factory
# an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
# backed by :memory: or a file)
res = factory(database, timeout, detect_types, isolation_level,
check_same_thread, factory, cached_statements, uri)
add_memory_pressure(100 * 1024)
return res
def push_event(self, time: Time, event_handler: Any) -> None:
"""
Push an event into the binary min heap.
Note that the 'insert' function implemented in C returns the number of bytes that are currently allocated for
all heap entries (a failed re-allocation is signaled by a return value of -1). If the PyPy interpreter is used,
and if the number of allocated bytes increased, one should add memory pressure by using the
'add_memory_pressure' function. If the CPython interpreter is used, this function does nothing (see
https://cffi.readthedocs.io/en/latest/using.html#memory-pressure-pypy). For the heap structure itself, this is
solved by using the 'size' keyword argument in the ffi.gc call (see '__init__' method of this class).
Also, the counter used for the lazy deletion is unbounded in Python. This might lead to the fact, that it cannot
be passed to the C function because C uses a bounded integer. If this is detected, all events from the
associated event handler are deleted in the heap (which necessarily have lower counter values) and the counter
is reset to 0.
Parameters
----------
time : base.time.Time
The event time.
event_handler : Any
The event handler.
Raises
------
MemoryError
If the C code fails to re-allocate memory for more heap entries.
"""
if time < inf:
if event_handler not in self._event_handler_handles.keys():
self._event_handler_handles[event_handler] = _new_handle(
event_handler)
try:
new_size = _lib_insert(
self._heap, time.quotient, time.remainder,
self._event_handler_handles[event_handler],
self._minimal_valid_counter.setdefault(event_handler, 0))
except OverflowError:
# Counter for event handler is too large to pass it to the C function.
_lib_delete_events(self._heap,
self._event_handler_handles[event_handler])
self._minimal_valid_counter[event_handler] = 0
new_size = _lib_insert(
self._heap, time.quotient, time.remainder,
self._event_handler_handles[event_handler], 0)
if new_size == self._allocated_memory_bytes:
return
elif new_size > self._allocated_memory_bytes:
add_memory_pressure(new_size - self._allocated_memory_bytes)
self._allocated_memory_bytes = new_size
elif new_size < self._allocated_memory_bytes:
raise MemoryError(
"Could not reallocate memory for the class {0}.".format(
self.__class__.__name__))
def _add_memory_pressure(p):
"""
PyPy's garbage collector is not aware of memory uses happening inside native code. When performing memory-intensive
tasks in native code, the memory pressure that PyPy observes can greatly deviate from the actual memory pressure.
We must manually add sufficient memory pressure to account for the "missing" portion.
This is not a problem for CPython since its GC is based on reference counting.
"""
global _is_pypy
if _is_pypy:
__pypy__.add_memory_pressure(p)
def _add_memory_pressure(p):
"""
PyPy's garbage collector is not aware of memory uses happening inside native code. When performing memory-intensive
tasks in native code, the memory pressure that PyPy observes can greatly deviate from the actual memory pressure.
We must manually add sufficient memory pressure to account for the "missing" portion.
This is not a problem for CPython since its GC is based on reference counting.
"""
global _is_pypy
if _is_pypy:
__pypy__.add_memory_pressure(p)
def validate_and_extract_assertion(token, key):
# type (str, str) -> JSONDict
"""Decode a web token into a assertion dictionary.
This attempts to rectify both ecdsa and openssl generated
signatures. We use the built-in cryptography library since it wraps
libssl and is faster than the python only approach.
:param token: VAPID auth token
:type token: str
:param key: bitarray containing public key
:type key: str or bitarray
:return dict of the VAPID claims
:raise InvalidSignature
"""
# convert the signature if needed.
try:
sig_material, signature = VerifyJWT.extract_signature(token)
pkey = ec.EllipticCurvePublicKey.from_encoded_point(
ec.SECP256R1(),
key
)
# cffi issue #320: public_key & verify allocate approx.
if _JWT_MEMORY_PRESSURE: # pragma: nocover
add_memory_pressure(_JWT_MEMORY_PRESSURE)
# NOTE: verify() will take any string as the signature. It appears
# to be doing lazy verification and matching strings rather than
# comparing content values. If the signatures start failing for
# some unknown reason in the future, decode the signature and
# make sure it matches how we're reconstructing it.
# This will raise an InvalidSignature exception if failure.
# It will be captured externally.
pkey.verify(
signature,
sig_material.encode('utf8'),
ec.ECDSA(hashes.SHA256()))
return VerifyJWT.extract_assertion(sig_material)
except InvalidSignature:
raise
except (ValueError, TypeError, binascii.Error, PyAsn1Error):
raise InvalidSignature()
except Exception: # pragma: no cover
Logger().failure("Unexpected error processing JWT")
raise InvalidSignature()
def __setstate__(self, state: MutableMapping[str, Any]) -> None:
"""
Use the state dictionary to initialize this class.
This method creates the _heap, _scheduler_handle, and _event_handler_handles attributes that were deleted in the
__getstate__ method. Moreover, all heap entries that were retrieved in the __getstate__ method are inserted into
the heap.
Parameters
----------
state : MutableMapping[str, Any]
The state.
Raises
------
MemoryError
If the C code fails to allocate memory.
"""
heap_entries = state["heap_entries"]
del state["heap_entries"]
self.__dict__.update(state)
c_heap = lib.construct_heap()
if c_heap == ffi.NULL:
raise MemoryError(
"Could not allocate memory for the class {0}.".format(
self.__class__.__name__))
self._heap = ffi.gc(c_heap,
lib.destroy_heap,
size=lib.estimated_size(c_heap))
new_size = 0
self._event_handler_handles = {}
for time_quotient, time_remainder, event_handler, counter in heap_entries:
if event_handler not in self._event_handler_handles.keys():
self._event_handler_handles[event_handler] = _new_handle(
event_handler)
new_size = _lib_insert(self._heap, time_quotient, time_remainder,
self._event_handler_handles[event_handler],
counter)
if new_size < 0:
raise MemoryError(
"Could not reallocate memory for the class {0}.".format(
self.__class__.__name__))
self._allocated_memory_bytes = new_size
add_memory_pressure(self._allocated_memory_bytes)
self._scheduler_handle = _new_handle(self)
def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None):
if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE):
raise ValueError("Integrity checks are only supported by FORMAT_XZ")
if preset is not None and filters is not None:
raise ValueError("Cannot specify both preset and filter chain")
if preset is None:
preset = m.LZMA_PRESET_DEFAULT
format = _parse_format(format)
self.lock = threading.Lock()
self.flushed = 0
self.lzs = _new_lzma_stream()
__pypy__.add_memory_pressure(COMPRESSION_STREAM_SIZE)
if format == FORMAT_XZ:
if filters is None:
if check == -1:
check = m.LZMA_CHECK_CRC64
catch_lzma_error(m.lzma_easy_encoder, self.lzs,
preset, check)
else:
filters = parse_filter_chain_spec(filters)
catch_lzma_error(m.lzma_stream_encoder, self.lzs,
filters, check)
elif format == FORMAT_ALONE:
if filters is None:
options = ffi.new('lzma_options_lzma*')
if m.lzma_lzma_preset(options, preset):
raise LZMAError("Invalid compression preset: %s" % preset)
catch_lzma_error(m.lzma_alone_encoder, self.lzs,
options)
else:
raise NotImplementedError
elif format == FORMAT_RAW:
if filters is None:
raise ValueError("Must specify filters for FORMAT_RAW")
filters = parse_filter_chain_spec(filters)
catch_lzma_error(m.lzma_raw_encoder, self.lzs,
filters)
else:
raise ValueError("invalid container format: %s" % format)
