class _UrlContextManager: def __init__(self, url, checksum_sha256): self.url = url self.checksum_sha256 = checksum_sha256 self.handle = None def __enter__(self): session = requests.Session() retry_policy = retry.Retry(connect=10, read=10, status=10, backoff_factor=0.1, status_forcelist=RETRY_STATUS_CODES) adapter = HTTPAdapter(max_retries=retry_policy) session.mount("http://", adapter) session.mount("https://", adapter) resp = session.get(self.url) resp.raise_for_status() self.handle = BytesIO(resp.content) verify_checksum(self.handle, self.checksum_sha256) return self.handle.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): try: return self.handle.__exit__(exc_type, exc_val, exc_tb) finally: self.handle = None
class _S3ContextManager: def __init__(self, s3_bucket, s3_key, checksum_sha256, s3_config): self.s3_bucket = s3_bucket self.s3_key = s3_key self.checksum_sha256 = checksum_sha256 self.s3_config = s3_config def __enter__(self): unsigned_requests = self.s3_config.get(S3Backend.CONFIG_UNSIGNED_REQUESTS_KEY, False) if unsigned_requests: resource_config = Config(signature_version=UNSIGNED) else: resource_config = None session = boto3.session.Session() s3 = session.resource("s3", config=resource_config) bucket = s3.Bucket(self.s3_bucket) s3_obj = bucket.Object(self.s3_key) self.buffer = BytesIO() s3_obj.download_fileobj(self.buffer) self.buffer.seek(0) verify_checksum(self.buffer, self.checksum_sha256) return self.buffer.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): try: return self.buffer.__exit__(exc_type, exc_val, exc_tb) finally: self.buffer = None
class _UrlContextManager(object): def __init__(self, url, checksum_sha256, seekable): self.url = url self.checksum_sha256 = checksum_sha256 self.seekable = seekable self.handle = None def __enter__(self): if self.seekable: req = requests.get(self.url) self.handle = BytesIO(req.content) else: req = requests.get(self.url, stream=True) self.handle = req.raw return self.handle.__enter__() def __exit__(self, exc_type, exc_val, exc_tb): if self.handle is not None: return self.handle.__exit__(exc_type, exc_val, exc_tb)
class _BaseBinaryWrapper: def __init__(self, stream: Union[typing.BinaryIO, bytes] = b""): if isinstance(stream, bytes) or isinstance(stream, bytearray): self.stream = BytesIO(stream) else: self.stream = stream # Wrappings: def close(self) -> None: return self.stream.close() def flush(self) -> None: return self.stream.flush() def read(self, n: int = -1) -> AnyStr: return self.stream.read(n) def readable(self) -> bool: return self.stream.readable() def readline(self, limit: int = -1) -> AnyStr: return self.stream.readline(limit) def readlines(self, hint: int = -1) -> List[AnyStr]: return self.stream.readlines(hint) def write(self, s: Union[bytes, bytearray]) -> int: return self.stream.write(s) def writable(self) -> bool: return self.stream.writable() def writelines(self, lines: Iterable[AnyStr]) -> None: self.stream.writelines(lines) def seek(self, offset: int, whence: int = 0) -> int: return self.stream.seek(offset, whence) def seekable(self) -> bool: return self.stream.seekable() def tell(self) -> int: return self.stream.tell() def fileno(self) -> int: return self.stream.fileno() def __enter__(self): self.stream.__enter__() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stream.__exit__(exc_type, exc_val, exc_tb) # helper functions def readall(self): self.stream.seek(0) return self.stream.read() def getvalue(self): if isinstance(self.stream, BytesIO): return self.stream.getvalue() pos = self.stream.tell() ret = self.readall() self.stream.seek(pos) return ret def align(self, alignment=4): if offset := (self.tell() % alignment): self.seek(self.tell() + alignment - offset)
class BCBytesStream(): """A class that provides additional serialization and deserialization methods over a base BufferedReader class. The BufferedReader object is class member _br and all unknown method calls are passed to _br""" def __init__(self, br=None): """Must be initialized with a BufferedReader.""" if type(br) == BufferedReader or type(br) == BytesIO: self._br = br elif type(br) == bytes: self._br = BytesIO(br) elif not br: self._br = BytesIO() else: raise TypeError( "BCBytesStream requires BufferedReader, BytesIO or bytes object, not {}" .format(type(br))) def __getattr__(self, name): return getattr(self._br, name) def __enter__(self, *args, **kwargs): self._br.__enter__(*args, **kwargs) return self def __exit__(self, *args, **kwargs): return self._br.__exit__(*args, **kwargs) def deser_boolean(self): return struct.unpack("?", self.read(1))[0] def ser_boolean(self, val): self.write(struct.pack("?", val)) def deser_int8(self): return struct.unpack("<b", self.read(1))[0] def ser_int8(self, val): self.write(struct.pack("<b", val)) def deser_uint8(self): return struct.unpack("<B", self.read(1))[0] def ser_uint8(self, val): self.write(struct.pack("<B", val)) def deser_int16(self): return struct.unpack("<h", self.read(2))[0] def ser_int16(self, val): self.write(struct.pack("<h", val)) def deser_uint16(self, big=False): fmt = ">" if big else "<" fmt += "H" return struct.unpack(fmt, self.read(2))[0] def ser_uint16(self, val): self.write(struct.pack("<H", val)) def deser_int32(self): return struct.unpack("<i", self.read(4))[0] def ser_int32(self, val): self.write(struct.pack("<i", val)) def deser_uint32(self): return struct.unpack("<I", self.read(4))[0] def ser_uint32(self, val): self.write(struct.pack("<I", val)) def deser_int64(self): return struct.unpack("<q", self.read(8))[0] def ser_int64(self, val): self.write(struct.pack("<q", val)) def deser_uint64(self): return struct.unpack("<Q", self.read(8))[0] def ser_uint64(self, val): self.write(struct.pack("<Q", val)) def deser_double(self): return struct.unpack("<d", self.read(8))[0] def ser_double(self, val): self.write(struct.pack("<d", val)) def deser_uint256(self): r = 0 for i in range(8): t = struct.unpack("<I", self.read(4))[0] r += t << (i * 32) return r def ser_uint256(self, val): rs = b"" for i in range(8): rs += struct.pack("<I", val & 0xFFFFFFFF) val >>= 32 self.write(rs) def deser_compact_size(self): nit = struct.unpack("<B", self.read(1))[0] if nit == 253: nit = struct.unpack("<H", self.read(2))[0] elif nit == 254: nit = struct.unpack("<I", self.read(4))[0] elif nit == 255: nit = struct.unpack("<Q", self.read(8))[0] return nit def ser_compact_size(self, val): r = b"" if val < 253: r = struct.pack("B", val) elif val < 0x10000: r = struct.pack("<BH", 253, val) elif val < 0x100000000: r = struct.pack("<BI", 254, val) else: r = struct.pack("<BQ", 255, val) self.write(r) def deser_string(self): nit = self.deser_compact_size() return self.read(nit).decode("utf-8") def ser_string(self, s): self.ser_compact_size(len(s)) self.write(s) def deser_vector(self, c): nit = self.deser_compact_size() r = [] for _ in range(nit): t = c() t.deserialize(self) r.append(t) return r def ser_vector(self, l, ser_function_name=None): """Serialize a vector object. ser_function_name: Allow for an alternate serialization function on the entries in the vector.""" self.ser_compact_size(len(l)) for i in l: if ser_function_name: getattr(i, ser_function_name)(self) else: self.write(i) def deser_uint256_vector(self): nit = self.deser_compact_size() r = [] for i in range(nit): t = self.deser_uint256(self) r.append(t) return r def ser_uint256_vector(self, l): self.ser_compact_size(len(l)) for i in l: self.ser_uint256(i) def deser_string_vector(self): nit = self.deser_compact_size() r = [] for i in range(nit): t = self.deser_string() r.append(t) return r def ser_string_vector(self, l): self.ser_compact_size(len(l)) for sv in l: self.ser_string(sv) def peep_byte(self): pos = self.tell() r = self.read(1) self.seek(pos) return r def deserialize_magic(self): magic = self.read(4) if magic == b'\xf9\xbe\xb4\xd9': network = "mainnet" elif magic == b'\x0b\x11\x09\x07': network = "testnet" elif magic == b'\xfa\xbf\xb5\xda': network = "regtest" else: network = "unknown" return magic, network