def test_pair_bin(self):
bs = bytearray(32)
buf = ArrayByteBuffer(bs)
writer = PairBinWriter(buf)
for i in range(10):
try:
writer.write(str(i).encode(), str(i).encode())
except IndexError as e:
print(buf.read_bytes(buf.get_offset(), 0))
buf.set_offset(0)
writer = PairBinWriter(buf)
writer.write(str(i).encode(), str(i).encode())
buf.set_offset(0)
reader = PairBinReader(buf)
print("last")
print(list(reader.read_all()))
def __init__(self, adapter: RollSiteAdapter, options: dict = None):
if options is None:
options = {}
self.adapter = adapter
self.roll_site_header: ErRollSiteHeader = adapter.roll_site_header
self.namespace = adapter.namespace
self.name = create_store_name(self.roll_site_header)
self.tagged_key = ''
self.obj_type = adapter.obj_type
self.proxy_endpoint = adapter.proxy_endpoint
channel = self.grpc_channel_factory.create_channel(self.proxy_endpoint)
self.stub = proxy_pb2_grpc.DataTransferServiceStub(channel)
static_er_conf = get_static_er_conf()
self.__bin_packet_len = int(options.get(
RollSiteConfKeys.EGGROLL_ROLLSITE_ADAPTER_SENDBUF_SIZE.key,
static_er_conf.get(RollSiteConfKeys.EGGROLL_ROLLSITE_ADAPTER_SENDBUF_SIZE.key,
RollSiteConfKeys.EGGROLL_ROLLSITE_ADAPTER_SENDBUF_SIZE.default_value)))
self.total_written = 0
self.ba = bytearray(self.__bin_packet_len)
self.buffer = ArrayByteBuffer(self.ba)
self.writer = PairBinWriter(pair_buffer=self.buffer)
self.push_batch_cnt = 0
self.push_pair_cnt = 0
self.topic_src = proxy_pb2.Topic(name=self.name, partyId=self.roll_site_header._src_party_id,
role=self.roll_site_header._src_role, callback=None)
self.topic_dst = proxy_pb2.Topic(name=self.name, partyId=self.roll_site_header._dst_party_id,
role=self.roll_site_header._dst_role, callback=None)
def test_pair_bin_no_abb(self):
bs = bytearray(32)
buf = ArrayByteBuffer(bs)
writer = PairBinWriter(pair_buffer=buf, data=bs)
for i in range(10):
try:
writer.write(str(i).encode(), str(i).encode())
except IndexError as e:
writer.set_offset(0)
writer = PairBinWriter(pair_buffer=buf, data=bs)
writer.write(str(i).encode(), str(i).encode())
pbr = PairBinReader(pair_buffer=buf, data=writer.get_data())
print(pbr.read_bytes(writer.get_offset(), 0))
writer.set_offset(0)
reader = PairBinReader(pair_buffer=buf, data=bs)
print("last")
print(list(reader.read_all()))
def commit(bs=sendbuf_size):
nonlocal ba
nonlocal buffer
nonlocal writer
bin_batch = None
if ba:
bin_batch = bytes(ba[0:buffer.get_offset()])
ba = bytearray(bs)
buffer = ArrayByteBuffer(ba)
writer = PairBinWriter(pair_buffer=buffer)
return bin_batch
def commit(bs=sendbuf_size):
L.debug(f'generate_bin_batch commit: {done_cnt}')
nonlocal ba
nonlocal buffer
nonlocal writer
bin_batch = None
if ba:
bin_batch = bytes(ba[0:buffer.get_offset()])
ba = bytearray(bs)
buffer = ArrayByteBuffer(ba)
writer = PairBinWriter(pair_buffer=buffer)
return bin_batch
def put(self, k, v):
if self.obj_type == 'object':
L.debug(f"set tagged_key: {k}")
self.tagged_key = _serdes.deserialize(k)
try:
self.writer.write(k, v)
except IndexError as e:
bin_batch = bytes(self.ba[0:self.buffer.get_offset()])
self.push(bin_batch)
# TODO:0: replace 1024 with constant
self.ba = bytearray(
max(self.__bin_packet_len,
len(k) + len(v) + 1024))
self.buffer = ArrayByteBuffer(self.ba)
self.writer = PairBinWriter(pair_buffer=self.buffer)
self.writer.write(k, v)
except Exception as e:
L.error(f"Unexpected error: {sys.exc_info()[0]}")
raise e
def __init__(self, file):
self._file = open(file.name, "wb")
self.writer = PairBinWriter(FileByteBuffer(self._file))
def __init__(self, file):
self.writer = PairBinWriter(FileByteBuffer(file))