class _UrlContextManager:
def __init__(self, url, checksum_sha256):
self.url = url
self.checksum_sha256 = checksum_sha256
self.handle = None
def __enter__(self):
session = requests.Session()
retry_policy = retry.Retry(connect=10,
read=10,
status=10,
backoff_factor=0.1,
status_forcelist=RETRY_STATUS_CODES)
adapter = HTTPAdapter(max_retries=retry_policy)
session.mount("http://", adapter)
session.mount("https://", adapter)
resp = session.get(self.url)
resp.raise_for_status()
self.handle = BytesIO(resp.content)
verify_checksum(self.handle, self.checksum_sha256)
return self.handle.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
try:
return self.handle.__exit__(exc_type, exc_val, exc_tb)
finally:
self.handle = None
class _S3ContextManager:
def __init__(self, s3_bucket, s3_key, checksum_sha256, s3_config):
self.s3_bucket = s3_bucket
self.s3_key = s3_key
self.checksum_sha256 = checksum_sha256
self.s3_config = s3_config
def __enter__(self):
unsigned_requests = self.s3_config.get(S3Backend.CONFIG_UNSIGNED_REQUESTS_KEY, False)
if unsigned_requests:
resource_config = Config(signature_version=UNSIGNED)
else:
resource_config = None
session = boto3.session.Session()
s3 = session.resource("s3", config=resource_config)
bucket = s3.Bucket(self.s3_bucket)
s3_obj = bucket.Object(self.s3_key)
self.buffer = BytesIO()
s3_obj.download_fileobj(self.buffer)
self.buffer.seek(0)
verify_checksum(self.buffer, self.checksum_sha256)
return self.buffer.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
try:
return self.buffer.__exit__(exc_type, exc_val, exc_tb)
finally:
self.buffer = None
class _UrlContextManager(object):
def __init__(self, url, checksum_sha256, seekable):
self.url = url
self.checksum_sha256 = checksum_sha256
self.seekable = seekable
self.handle = None
def __enter__(self):
if self.seekable:
req = requests.get(self.url)
self.handle = BytesIO(req.content)
else:
req = requests.get(self.url, stream=True)
self.handle = req.raw
return self.handle.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
if self.handle is not None:
return self.handle.__exit__(exc_type, exc_val, exc_tb)
class _BaseBinaryWrapper:
def __init__(self, stream: Union[typing.BinaryIO, bytes] = b""):
if isinstance(stream, bytes) or isinstance(stream, bytearray):
self.stream = BytesIO(stream)
else:
self.stream = stream
# Wrappings:
def close(self) -> None:
return self.stream.close()
def flush(self) -> None:
return self.stream.flush()
def read(self, n: int = -1) -> AnyStr:
return self.stream.read(n)
def readable(self) -> bool:
return self.stream.readable()
def readline(self, limit: int = -1) -> AnyStr:
return self.stream.readline(limit)
def readlines(self, hint: int = -1) -> List[AnyStr]:
return self.stream.readlines(hint)
def write(self, s: Union[bytes, bytearray]) -> int:
return self.stream.write(s)
def writable(self) -> bool:
return self.stream.writable()
def writelines(self, lines: Iterable[AnyStr]) -> None:
self.stream.writelines(lines)
def seek(self, offset: int, whence: int = 0) -> int:
return self.stream.seek(offset, whence)
def seekable(self) -> bool:
return self.stream.seekable()
def tell(self) -> int:
return self.stream.tell()
def fileno(self) -> int:
return self.stream.fileno()
def __enter__(self):
self.stream.__enter__()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stream.__exit__(exc_type, exc_val, exc_tb)
# helper functions
def readall(self):
self.stream.seek(0)
return self.stream.read()
def getvalue(self):
if isinstance(self.stream, BytesIO):
return self.stream.getvalue()
pos = self.stream.tell()
ret = self.readall()
self.stream.seek(pos)
return ret
def align(self, alignment=4):
if offset := (self.tell() % alignment):
self.seek(self.tell() + alignment - offset)
class BCBytesStream():
"""A class that provides additional serialization and deserialization methods
over a base BufferedReader class.
The BufferedReader object is class member _br and all unknown method
calls are passed to _br"""
def __init__(self, br=None):
"""Must be initialized with a BufferedReader."""
if type(br) == BufferedReader or type(br) == BytesIO:
self._br = br
elif type(br) == bytes:
self._br = BytesIO(br)
elif not br:
self._br = BytesIO()
else:
raise TypeError(
"BCBytesStream requires BufferedReader, BytesIO or bytes object, not {}"
.format(type(br)))
def __getattr__(self, name):
return getattr(self._br, name)
def __enter__(self, *args, **kwargs):
self._br.__enter__(*args, **kwargs)
return self
def __exit__(self, *args, **kwargs):
return self._br.__exit__(*args, **kwargs)
def deser_boolean(self):
return struct.unpack("?", self.read(1))[0]
def ser_boolean(self, val):
self.write(struct.pack("?", val))
def deser_int8(self):
return struct.unpack("<b", self.read(1))[0]
def ser_int8(self, val):
self.write(struct.pack("<b", val))
def deser_uint8(self):
return struct.unpack("<B", self.read(1))[0]
def ser_uint8(self, val):
self.write(struct.pack("<B", val))
def deser_int16(self):
return struct.unpack("<h", self.read(2))[0]
def ser_int16(self, val):
self.write(struct.pack("<h", val))
def deser_uint16(self, big=False):
fmt = ">" if big else "<"
fmt += "H"
return struct.unpack(fmt, self.read(2))[0]
def ser_uint16(self, val):
self.write(struct.pack("<H", val))
def deser_int32(self):
return struct.unpack("<i", self.read(4))[0]
def ser_int32(self, val):
self.write(struct.pack("<i", val))
def deser_uint32(self):
return struct.unpack("<I", self.read(4))[0]
def ser_uint32(self, val):
self.write(struct.pack("<I", val))
def deser_int64(self):
return struct.unpack("<q", self.read(8))[0]
def ser_int64(self, val):
self.write(struct.pack("<q", val))
def deser_uint64(self):
return struct.unpack("<Q", self.read(8))[0]
def ser_uint64(self, val):
self.write(struct.pack("<Q", val))
def deser_double(self):
return struct.unpack("<d", self.read(8))[0]
def ser_double(self, val):
self.write(struct.pack("<d", val))
def deser_uint256(self):
r = 0
for i in range(8):
t = struct.unpack("<I", self.read(4))[0]
r += t << (i * 32)
return r
def ser_uint256(self, val):
rs = b""
for i in range(8):
rs += struct.pack("<I", val & 0xFFFFFFFF)
val >>= 32
self.write(rs)
def deser_compact_size(self):
nit = struct.unpack("<B", self.read(1))[0]
if nit == 253:
nit = struct.unpack("<H", self.read(2))[0]
elif nit == 254:
nit = struct.unpack("<I", self.read(4))[0]
elif nit == 255:
nit = struct.unpack("<Q", self.read(8))[0]
return nit
def ser_compact_size(self, val):
r = b""
if val < 253:
r = struct.pack("B", val)
elif val < 0x10000:
r = struct.pack("<BH", 253, val)
elif val < 0x100000000:
r = struct.pack("<BI", 254, val)
else:
r = struct.pack("<BQ", 255, val)
self.write(r)
def deser_string(self):
nit = self.deser_compact_size()
return self.read(nit).decode("utf-8")
def ser_string(self, s):
self.ser_compact_size(len(s))
self.write(s)
def deser_vector(self, c):
nit = self.deser_compact_size()
r = []
for _ in range(nit):
t = c()
t.deserialize(self)
r.append(t)
return r
def ser_vector(self, l, ser_function_name=None):
"""Serialize a vector object.
ser_function_name: Allow for an alternate serialization function on the
entries in the vector."""
self.ser_compact_size(len(l))
for i in l:
if ser_function_name:
getattr(i, ser_function_name)(self)
else:
self.write(i)
def deser_uint256_vector(self):
nit = self.deser_compact_size()
r = []
for i in range(nit):
t = self.deser_uint256(self)
r.append(t)
return r
def ser_uint256_vector(self, l):
self.ser_compact_size(len(l))
for i in l:
self.ser_uint256(i)
def deser_string_vector(self):
nit = self.deser_compact_size()
r = []
for i in range(nit):
t = self.deser_string()
r.append(t)
return r
def ser_string_vector(self, l):
self.ser_compact_size(len(l))
for sv in l:
self.ser_string(sv)
def peep_byte(self):
pos = self.tell()
r = self.read(1)
self.seek(pos)
return r
def deserialize_magic(self):
magic = self.read(4)
if magic == b'\xf9\xbe\xb4\xd9':
network = "mainnet"
elif magic == b'\x0b\x11\x09\x07':
network = "testnet"
elif magic == b'\xfa\xbf\xb5\xda':
network = "regtest"
else:
network = "unknown"
return magic, network
