def flush(self): with self.lock: if self.flushed: raise ValueError("Repeated call to flush()") self.flushed = 1 result = self._compress(b'', action=m.LZMA_FINISH) __pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE) _release_lzma_stream(self.lzs) return result
def connect(database, timeout=5.0, detect_types=0, isolation_level="", check_same_thread=True, factory=None, cached_statements=100, uri=0): factory = Connection if not factory else factory # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if # backed by :memory: or a file) res = factory(database, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri) add_memory_pressure(100 * 1024) return res
def push_event(self, time: Time, event_handler: Any) -> None: """ Push an event into the binary min heap. Note that the 'insert' function implemented in C returns the number of bytes that are currently allocated for all heap entries (a failed re-allocation is signaled by a return value of -1). If the PyPy interpreter is used, and if the number of allocated bytes increased, one should add memory pressure by using the 'add_memory_pressure' function. If the CPython interpreter is used, this function does nothing (see https://cffi.readthedocs.io/en/latest/using.html#memory-pressure-pypy). For the heap structure itself, this is solved by using the 'size' keyword argument in the ffi.gc call (see '__init__' method of this class). Also, the counter used for the lazy deletion is unbounded in Python. This might lead to the fact, that it cannot be passed to the C function because C uses a bounded integer. If this is detected, all events from the associated event handler are deleted in the heap (which necessarily have lower counter values) and the counter is reset to 0. Parameters ---------- time : base.time.Time The event time. event_handler : Any The event handler. Raises ------ MemoryError If the C code fails to re-allocate memory for more heap entries. """ if time < inf: if event_handler not in self._event_handler_handles.keys(): self._event_handler_handles[event_handler] = _new_handle( event_handler) try: new_size = _lib_insert( self._heap, time.quotient, time.remainder, self._event_handler_handles[event_handler], self._minimal_valid_counter.setdefault(event_handler, 0)) except OverflowError: # Counter for event handler is too large to pass it to the C function. _lib_delete_events(self._heap, self._event_handler_handles[event_handler]) self._minimal_valid_counter[event_handler] = 0 new_size = _lib_insert( self._heap, time.quotient, time.remainder, self._event_handler_handles[event_handler], 0) if new_size == self._allocated_memory_bytes: return elif new_size > self._allocated_memory_bytes: add_memory_pressure(new_size - self._allocated_memory_bytes) self._allocated_memory_bytes = new_size elif new_size < self._allocated_memory_bytes: raise MemoryError( "Could not reallocate memory for the class {0}.".format( self.__class__.__name__))
def _add_memory_pressure(p): """ PyPy's garbage collector is not aware of memory uses happening inside native code. When performing memory-intensive tasks in native code, the memory pressure that PyPy observes can greatly deviate from the actual memory pressure. We must manually add sufficient memory pressure to account for the "missing" portion. This is not a problem for CPython since its GC is based on reference counting. """ global _is_pypy if _is_pypy: __pypy__.add_memory_pressure(p)
def _add_memory_pressure(p): """ PyPy's garbage collector is not aware of memory uses happening inside native code. When performing memory-intensive tasks in native code, the memory pressure that PyPy observes can greatly deviate from the actual memory pressure. We must manually add sufficient memory pressure to account for the "missing" portion. This is not a problem for CPython since its GC is based on reference counting. """ global _is_pypy if _is_pypy: __pypy__.add_memory_pressure(p)
def validate_and_extract_assertion(token, key): # type (str, str) -> JSONDict """Decode a web token into a assertion dictionary. This attempts to rectify both ecdsa and openssl generated signatures. We use the built-in cryptography library since it wraps libssl and is faster than the python only approach. :param token: VAPID auth token :type token: str :param key: bitarray containing public key :type key: str or bitarray :return dict of the VAPID claims :raise InvalidSignature """ # convert the signature if needed. try: sig_material, signature = VerifyJWT.extract_signature(token) pkey = ec.EllipticCurvePublicKey.from_encoded_point( ec.SECP256R1(), key ) # cffi issue #320: public_key & verify allocate approx. if _JWT_MEMORY_PRESSURE: # pragma: nocover add_memory_pressure(_JWT_MEMORY_PRESSURE) # NOTE: verify() will take any string as the signature. It appears # to be doing lazy verification and matching strings rather than # comparing content values. If the signatures start failing for # some unknown reason in the future, decode the signature and # make sure it matches how we're reconstructing it. # This will raise an InvalidSignature exception if failure. # It will be captured externally. pkey.verify( signature, sig_material.encode('utf8'), ec.ECDSA(hashes.SHA256())) return VerifyJWT.extract_assertion(sig_material) except InvalidSignature: raise except (ValueError, TypeError, binascii.Error, PyAsn1Error): raise InvalidSignature() except Exception: # pragma: no cover Logger().failure("Unexpected error processing JWT") raise InvalidSignature()
def __setstate__(self, state: MutableMapping[str, Any]) -> None: """ Use the state dictionary to initialize this class. This method creates the _heap, _scheduler_handle, and _event_handler_handles attributes that were deleted in the __getstate__ method. Moreover, all heap entries that were retrieved in the __getstate__ method are inserted into the heap. Parameters ---------- state : MutableMapping[str, Any] The state. Raises ------ MemoryError If the C code fails to allocate memory. """ heap_entries = state["heap_entries"] del state["heap_entries"] self.__dict__.update(state) c_heap = lib.construct_heap() if c_heap == ffi.NULL: raise MemoryError( "Could not allocate memory for the class {0}.".format( self.__class__.__name__)) self._heap = ffi.gc(c_heap, lib.destroy_heap, size=lib.estimated_size(c_heap)) new_size = 0 self._event_handler_handles = {} for time_quotient, time_remainder, event_handler, counter in heap_entries: if event_handler not in self._event_handler_handles.keys(): self._event_handler_handles[event_handler] = _new_handle( event_handler) new_size = _lib_insert(self._heap, time_quotient, time_remainder, self._event_handler_handles[event_handler], counter) if new_size < 0: raise MemoryError( "Could not reallocate memory for the class {0}.".format( self.__class__.__name__)) self._allocated_memory_bytes = new_size add_memory_pressure(self._allocated_memory_bytes) self._scheduler_handle = _new_handle(self)
def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None): if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE): raise ValueError("Integrity checks are only supported by FORMAT_XZ") if preset is not None and filters is not None: raise ValueError("Cannot specify both preset and filter chain") if preset is None: preset = m.LZMA_PRESET_DEFAULT format = _parse_format(format) self.lock = threading.Lock() self.flushed = 0 self.lzs = _new_lzma_stream() __pypy__.add_memory_pressure(COMPRESSION_STREAM_SIZE) if format == FORMAT_XZ: if filters is None: if check == -1: check = m.LZMA_CHECK_CRC64 catch_lzma_error(m.lzma_easy_encoder, self.lzs, preset, check) else: filters = parse_filter_chain_spec(filters) catch_lzma_error(m.lzma_stream_encoder, self.lzs, filters, check) elif format == FORMAT_ALONE: if filters is None: options = ffi.new('lzma_options_lzma*') if m.lzma_lzma_preset(options, preset): raise LZMAError("Invalid compression preset: %s" % preset) catch_lzma_error(m.lzma_alone_encoder, self.lzs, options) else: raise NotImplementedError elif format == FORMAT_RAW: if filters is None: raise ValueError("Must specify filters for FORMAT_RAW") filters = parse_filter_chain_spec(filters) catch_lzma_error(m.lzma_raw_encoder, self.lzs, filters) else: raise ValueError("invalid container format: %s" % format)