def __init__(self, space, name, copy_from=NULL_CTX):
self.name = name
digest_type = self.digest_type_by_name(space)
self.digest_size = ropenssl.EVP_MD_size(digest_type)
# Allocate a lock for each HASH object.
# An optimization would be to not release the GIL on small requests,
# and use a custom lock only when needed.
self.lock = Lock(space)
ctx = ropenssl.EVP_MD_CTX_new()
if ctx is None:
raise MemoryError
rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size,
self)
try:
if copy_from:
if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
raise ValueError
else:
ropenssl.EVP_DigestInit(ctx, digest_type)
self.ctx = ctx
except:
ropenssl.EVP_MD_CTX_free(ctx)
raise
self.register_finalizer(space)
def deflateInit(level=Z_DEFAULT_COMPRESSION,
method=Z_DEFLATED,
wbits=MAX_WBITS,
memLevel=DEF_MEM_LEVEL,
strategy=Z_DEFAULT_STRATEGY,
zdict=None):
"""
Allocate and return an opaque 'stream' object that can be used to
compress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
if err == Z_OK:
if zdict is not None:
try:
deflateSetDictionary(stream, zdict)
except:
lltype.free(stream, flavor='raw')
raise
return stream
else:
try:
if err == Z_STREAM_ERROR:
raise ValueError("Invalid initialization option")
else:
raise RZlibError.fromstream(
stream, err, "while creating compression object")
finally:
lltype.free(stream, flavor='raw')
def inflateInit(wbits=MAX_WBITS, zdict=None):
"""
Allocate and return an opaque 'stream' object that can be used to
decompress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _inflateInit2(stream, wbits)
if err == Z_OK:
if zdict is not None and wbits < 0:
try:
if ZLIB_VERNUM is None or ZLIB_VERNUM < 0x1221:
raise RZlibError("zlib version %s does not allow raw "
"inflate with dictionary" %
ZLIB_VERSION if ZLIB_VERSION is not None
else "<unknown>")
inflateSetDictionary(stream, zdict)
except:
lltype.free(stream, flavor='raw')
raise
return stream
else:
try:
if err == Z_STREAM_ERROR:
raise ValueError("Invalid initialization option")
else:
raise RZlibError.fromstream(
stream, err, "while creating decompression object")
finally:
lltype.free(stream, flavor='raw')
def _do_exit(self):
w_free = self.w_free
if w_free is not None:
rgc.add_memory_pressure(-self._sizeof(), self)
self.w_free = None
self.may_unregister_rpython_finalizer(self.space)
self.space.call_function(w_free, self.w_raw_cdata)
def ParserCreate(space,
w_encoding=None,
w_namespace_separator=None,
w_intern=None):
"""ParserCreate([encoding[, namespace_separator]]) -> parser
Return a new XML parser object."""
if space.is_none(w_encoding):
encoding = None
elif space.isinstance_w(w_encoding, space.w_str):
encoding = space.str_w(w_encoding)
else:
raise oefmt(
space.w_TypeError,
"ParserCreate() argument 1 must be string or None, not %T",
w_encoding)
if space.is_none(w_namespace_separator):
namespace_separator = 0
elif space.isinstance_w(w_namespace_separator, space.w_str):
separator = space.str_w(w_namespace_separator)
if len(separator) == 0:
namespace_separator = 0
elif len(separator) == 1:
namespace_separator = ord(separator[0])
else:
raise OperationError(
space.w_ValueError,
space.wrap('namespace_separator must be at most one character,'
' omitted, or None'))
else:
raise oefmt(
space.w_TypeError,
"ParserCreate() argument 2 must be string or None, not %T",
w_namespace_separator)
# Explicitly passing None means no interning is desired.
# Not passing anything means that a new dictionary is used.
if w_intern is None:
w_intern = space.newdict()
elif space.is_w(w_intern, space.w_None):
w_intern = None
if namespace_separator:
xmlparser = XML_ParserCreateNS(
encoding, rffi.cast(rffi.CHAR, namespace_separator))
else:
xmlparser = XML_ParserCreate(encoding)
# Currently this is just the size of the pointer and some estimated bytes.
# The struct isn't actually defined in expat.h - it is in xmlparse.c
# XXX: find a good estimate of the XML_ParserStruct
rgc.add_memory_pressure(XML_Parser_SIZE + 300)
if not xmlparser:
raise OperationError(space.w_RuntimeError,
space.wrap('XML_ParserCreate failed'))
parser = W_XMLParserType(space, xmlparser, w_intern)
XML_SetUnknownEncodingHandler(parser.itself,
UnknownEncodingHandlerData_callback,
rffi.cast(rffi.VOIDP, parser.id))
return space.wrap(parser)
def __init__(self, space, name, copy_from=NULL_CTX):
self.name = name
digest_type = self.digest_type_by_name(space)
self.digest_size = ropenssl.EVP_MD_size(digest_type)
# Allocate a lock for each HASH object.
# An optimization would be to not release the GIL on small requests,
# and use a custom lock only when needed.
self.lock = Lock(space)
ctx = ropenssl.EVP_MD_CTX_new()
if ctx is None:
raise MemoryError
rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
try:
if copy_from:
if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
raise ValueError
else:
ropenssl.EVP_DigestInit(ctx, digest_type)
self.ctx = ctx
except:
ropenssl.EVP_MD_CTX_free(ctx)
raise
self.register_finalizer(space)
def ParserCreate(space, w_encoding=None, w_namespace_separator=None,
w_intern=None):
"""ParserCreate([encoding[, namespace_separator]]) -> parser
Return a new XML parser object."""
if space.is_none(w_encoding):
encoding = None
elif space.isinstance_w(w_encoding, space.w_str):
encoding = space.str_w(w_encoding)
else:
raise operationerrfmt(
space.w_TypeError,
'ParserCreate() argument 1 must be string or None, not %T',
w_encoding)
if space.is_none(w_namespace_separator):
namespace_separator = 0
elif space.isinstance_w(w_namespace_separator, space.w_str):
separator = space.str_w(w_namespace_separator)
if len(separator) == 0:
namespace_separator = 0
elif len(separator) == 1:
namespace_separator = ord(separator[0])
else:
raise OperationError(
space.w_ValueError,
space.wrap('namespace_separator must be at most one character,'
' omitted, or None'))
else:
raise operationerrfmt(
space.w_TypeError,
'ParserCreate() argument 2 must be string or None, not %T',
w_namespace_separator)
# Explicitly passing None means no interning is desired.
# Not passing anything means that a new dictionary is used.
if w_intern is None:
w_intern = space.newdict()
elif space.is_w(w_intern, space.w_None):
w_intern = None
if namespace_separator:
xmlparser = XML_ParserCreateNS(
encoding,
rffi.cast(rffi.CHAR, namespace_separator))
else:
xmlparser = XML_ParserCreate(encoding)
# Currently this is just the size of the pointer and some estimated bytes.
# The struct isn't actually defined in expat.h - it is in xmlparse.c
# XXX: find a good estimate of the XML_ParserStruct
rgc.add_memory_pressure(XML_Parser_SIZE + 300)
if not xmlparser:
raise OperationError(space.w_RuntimeError,
space.wrap('XML_ParserCreate failed'))
parser = W_XMLParserType(space, xmlparser, w_intern)
XML_SetUnknownEncodingHandler(
parser.itself, UnknownEncodingHandlerData_callback,
rffi.cast(rffi.VOIDP, parser.id))
return space.wrap(parser)
def create_semaphore(space, name, val, max):
sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
try:
sem_unlink(name)
except OSError:
pass
else:
rgc.add_memory_pressure(SEM_T_SIZE)
return sem
def create_semaphore(space, name, val, max):
sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
try:
sem_unlink(name)
except OSError:
pass
else:
rgc.add_memory_pressure(SEM_T_SIZE)
return sem
def allocate_ll_lock():
# track_allocation=False here; be careful to lltype.free() it. The
# reason it is set to False is that we get it from all app-level
# lock objects, as well as from the GIL, which exists at shutdown.
ll_lock = lltype.malloc(TLOCKP.TO, flavor='raw', track_allocation=False)
res = c_thread_lock_init(ll_lock)
if rffi.cast(lltype.Signed, res) <= 0:
lltype.free(ll_lock, flavor='raw', track_allocation=False)
raise error("out of resources")
# Add some memory pressure for the size of the lock because it is an
# Opaque object
rgc.add_memory_pressure(TLOCKP_SIZE)
return ll_lock
def setlen(self, size, zero=False, overallocate=True):
if self._buffer:
delta_memory_pressure = -self.allocated * self.itemsize
else:
delta_memory_pressure = 0
if size > 0:
if size > self.allocated or size < self.allocated / 2:
if overallocate:
if size < 9:
some = 3
else:
some = 6
some += size >> 3
else:
some = 0
self.allocated = size + some
byte_size = self.allocated * self.itemsize
delta_memory_pressure += byte_size
if zero:
new_buffer = lltype.malloc(rffi.CCHARP.TO,
byte_size,
flavor='raw',
zero=True)
else:
new_buffer = lltype.malloc(rffi.CCHARP.TO,
byte_size,
flavor='raw')
copy_bytes = min(size, self.len) * self.itemsize
rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
rffi.cast(rffi.VOIDP, self._buffer),
copy_bytes)
else:
self.len = size
return
else:
assert size == 0
self.allocated = 0
new_buffer = lltype.nullptr(rffi.CCHARP.TO)
if self._buffer:
lltype.free(self._buffer, flavor='raw')
self._buffer = new_buffer
self.len = size
# adds the difference between the old and the new raw-malloced
# size. If setlen() is called a lot on the same array object,
# it is important to take into account the fact that we also do
# lltype.free() above.
rgc.add_memory_pressure(delta_memory_pressure)
def with_gc(self, w_destructor, size=0):
space = self.space
if space.is_none(w_destructor):
if isinstance(self, W_CDataGCP):
self.detach_destructor()
w_res = space.w_None
else:
raise oefmt(
space.w_TypeError,
"Can remove destructor only on a object "
"previously returned by ffi.gc()")
else:
with self as ptr:
w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
if size != 0:
rgc.add_memory_pressure(size)
return w_res
def inflateInit(wbits=MAX_WBITS):
"""
Allocate and return an opaque 'stream' object that can be used to
decompress data.
"""
stream = lltype.malloc(z_stream, flavor="raw", zero=True)
rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _inflateInit2(stream, wbits)
if err == Z_OK:
return stream
else:
try:
if err == Z_STREAM_ERROR:
raise ValueError("Invalid initialization option")
else:
raise RZlibError.fromstream(stream, err, "while creating decompression object")
finally:
lltype.free(stream, flavor="raw")
def allocate(self, space, datasize, ctype, length=-1):
from pypy.module._cffi_backend import cdataobj, ctypeptr
if self.w_alloc is None:
if self.should_clear_after_alloc:
ptr = lltype.malloc(rffi.CCHARP.TO,
datasize,
flavor='raw',
zero=True,
add_memory_pressure=True)
else:
ptr = lltype.malloc(rffi.CCHARP.TO,
datasize,
flavor='raw',
zero=False,
add_memory_pressure=True)
return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
else:
w_raw_cdata = space.call_function(self.w_alloc,
space.newint(datasize))
if not isinstance(w_raw_cdata, cdataobj.W_CData):
raise oefmt(space.w_TypeError,
"alloc() must return a cdata object (got %T)",
w_raw_cdata)
if not isinstance(w_raw_cdata.ctype, ctypeptr.W_CTypePtrOrArray):
raise oefmt(space.w_TypeError,
"alloc() must return a cdata pointer, not '%s'",
w_raw_cdata.ctype.name)
#
ptr = w_raw_cdata.unsafe_escaping_ptr()
if not ptr:
raise oefmt(space.w_MemoryError, "alloc() returned NULL")
#
if self.should_clear_after_alloc:
rffi.c_memset(rffi.cast(rffi.VOIDP, ptr), 0,
rffi.cast(rffi.SIZE_T, datasize))
#
res = cdataobj.W_CDataNewNonStd(space, ptr, ctype, length)
res.w_raw_cdata = w_raw_cdata
if self.w_free is not None:
res.w_free = self.w_free
res.register_finalizer(space)
rgc.add_memory_pressure(datasize)
return res
def inflateInit(wbits=MAX_WBITS):
"""
Allocate and return an opaque 'stream' object that can be used to
decompress data.
"""
stream = lltype.malloc(z_stream, flavor='raw', zero=True)
rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _inflateInit2(stream, wbits)
if err == Z_OK:
return stream
else:
try:
if err == Z_STREAM_ERROR:
raise ValueError("Invalid initialization option")
else:
raise RZlibError.fromstream(
stream, err, "while creating decompression object")
finally:
lltype.free(stream, flavor='raw')
def deflateInit(
level=Z_DEFAULT_COMPRESSION, method=Z_DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY
):
"""
Allocate and return an opaque 'stream' object that can be used to
compress data.
"""
stream = lltype.malloc(z_stream, flavor="raw", zero=True)
rgc.add_memory_pressure(rffi.sizeof(z_stream))
err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
if err == Z_OK:
return stream
else:
try:
if err == Z_STREAM_ERROR:
raise ValueError("Invalid initialization option")
else:
raise RZlibError.fromstream(stream, err, "while creating compression object")
finally:
lltype.free(stream, flavor="raw")
def __init__(self, space, name, copy_from=NULL_CTX):
self.name = name
digest_type = self.digest_type_by_name(space)
self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
# Allocate a lock for each HASH object.
# An optimization would be to not release the GIL on small requests,
# and use a custom lock only when needed.
self.lock = Lock(space)
ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
try:
if copy_from:
ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
else:
ropenssl.EVP_DigestInit(ctx, digest_type)
self.ctx = ctx
except:
lltype.free(ctx, flavor='raw')
raise
def __init__(self, space, name, copy_from=NULL_CTX):
self.name = name
digest_type = self.digest_type_by_name(space)
self.digest_size = rffi.getintfield(digest_type, 'c_md_size')
# Allocate a lock for each HASH object.
# An optimization would be to not release the GIL on small requests,
# and use a custom lock only when needed.
self.lock = Lock(space)
ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO, flavor='raw')
rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
try:
if copy_from:
ropenssl.EVP_MD_CTX_copy(ctx, copy_from)
else:
ropenssl.EVP_DigestInit(ctx, digest_type)
self.ctx = ctx
except:
lltype.free(ctx, flavor='raw')
raise
def add_memory_pressure(space, estimate):
""" Add memory pressure of estimate bytes. Useful when calling a C function
that internally allocates a big chunk of memory. This instructs the GC to
garbage collect sooner than it would otherwise."""
rgc.add_memory_pressure(estimate)
def allocate_lock():
# Add some memory pressure for the size of the lock because it is an
# Opaque object
lock = Lock(allocate_ll_lock())
rgc.add_memory_pressure(TLOCKP_SIZE, lock)
return lock
def reopen_semaphore(name):
sem = sem_open(name, 0, 0600, 0)
rgc.add_memory_pressure(SEM_T_SIZE)
return sem
def create_semaphore(space, name, val, max):
sem = sem_open(name, os.O_CREAT | os.O_EXCL, 0600, val)
rgc.add_memory_pressure(SEM_T_SIZE)
return sem
def __init__(self):
self.ctx = lltype.malloc(ropenssl.EVP_MD_CTX.TO,
flavor='raw')
digest = ropenssl.EVP_get_digestbyname('sha1')
ropenssl.EVP_DigestInit(self.ctx, digest)
rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
def add_memory_pressure(estimate):
rgc.add_memory_pressure(estimate)
def add_memory_pressure(estimate):
rgc.add_memory_pressure(estimate)
def _do_exit(self):
if self.datasize >= 0:
rgc.add_memory_pressure(-self.datasize, self)
self.datasize = -1
rgc.may_ignore_finalizer(self)
lltype.free(self._ptr, flavor='raw')
def __init__(self, length):
self._length = length
self._buf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw', zero=True)
rgc.add_memory_pressure(length)
self.readonly = False
def _PyPyGC_AddMemoryPressure(space, report):
from rpython.rlib import rgc
rgc.add_memory_pressure(report)
def _do_exit(self):
if not self.explicitly_freed:
rgc.add_memory_pressure(-self._sizeof(), self)
self.explicitly_freed = True
rgc.may_ignore_finalizer(self)
lltype.free(self._ptr, flavor='raw')