def generate_broker():
with create_adapter(task._inputs[0]) as db, db.iteritems() as rb:
limit = None if er_pair._key is None else key_serdes.deserialize(er_pair._key)
try:
yield from TransferPair.pair_to_bin_batch(rb, limit=limit)
finally:
TransferService.remove_broker(tag)
def _generate_batch_streams(self, pair_iter, batches_per_stream,
body_bytes):
batches = TransferPair.pair_to_bin_batch(pair_iter,
sendbuf_size=body_bytes)
try:
peek = next(batches)
except StopIteration as e:
self._finish_partition = True
def chunk_batch_stream():
nonlocal self
nonlocal peek
cur_batch = peek
try:
for i in range(batches_per_stream - 1):
next_batch = next(batches)
yield cur_batch
cur_batch = next_batch
peek = next(batches)
except StopIteration as e:
self._finish_partition = True
finally:
yield cur_batch
try:
while not self._finish_partition:
self._rs_header._stream_seq += 1
yield chunk_batch_stream()
except Exception as e:
L.exception(
f'error in generating stream, rs_key={self._rs_header.get_rs_key()}, rs_header={self._rs_header}'
)
def write(self):
L.info("RemoteRollPairWriteBatch write calling")
if len(self.manual_merger) == 0:
L.info(f"self.manual_merger={self.manual_merger}")
return
self.has_write_op = True
batches = TransferPair.pair_to_bin_batch(
sorted(self.manual_merger.items(), key=lambda kv: kv[0]))
task_id = f"{self.adapter._replicate_job_id}-partition-{self.adapter._partition_id}"
L.info(f"task_id={task_id}")
tasks = [
ErTask(id=task_id,
name=RollPair.PUT_BATCH,
inputs=[self.adapter._er_partition],
outputs=[self.adapter._er_partition],
job=ErJob(id=self.adapter._replicate_job_id,
name=RollPair.PUT_BATCH))
]
def send_command(tasks, remote_cmd_endpoint):
cmd_client = CommandClient()
return cmd_client.sync_send(
inputs=tasks,
output_types=[ErTask],
endpoint=remote_cmd_endpoint,
command_uri=CommandURI(f'v1/egg-pair/runTask'))
L.info(f"start to send cmd")
t = Thread(target=send_command,
name=task_id,
args=[tasks, self.adapter.remote_cmd_endpoint])
t.start()
transfer_client = TransferClient()
f = transfer_client.send(
batches,
endpoint=self.adapter.remote_transfer_endpoint,
tag=task_id)
f.result()
t.join()
self.manual_merger.clear()
L.info("RemoteRollPairWriteBatch write called")
def generate_broker():
with create_adapter(
task._inputs[0]) as db, db.iteritems() as rb:
yield from TransferPair.pair_to_bin_batch(rb)