def __init__(self, options: dict = None):
if options is None:
options = {}
super().__init__(options)
er_partition = options['er_partition']
self.partition = er_partition
self.store_locator = er_partition._store_locator
self.partition_id = er_partition._id
self.namespace = self.store_locator._namespace
#_store_type = StoreTypes.ROLLPAIR_ROLLSITE
# self._store_locator = meta_pb2.StoreLocator(storeType=_store_type,
# namespace=self.namespace,
# name=self.store_locator._name,
# partitioner=self.store_locator._partitioner,
# serdes=self.store_locator._serdes,
# totalPartitions=self.store_locator._total_partitions)
self.roll_site_header_string = options.get('roll_site_header', None)
self.is_writable = False
if self.roll_site_header_string:
self.roll_site_header = ErRollSiteHeader.from_proto_string(
self.roll_site_header_string.encode(stringify_charset))
self.proxy_endpoint = ErEndpoint.from_proto_string(
options['proxy_endpoint'].encode(stringify_charset))
self.obj_type = options['obj_type']
self.is_writable = True
L.info(
f"writable RollSiteAdapter: {self.namespace}, {self.partition_id}. proxy_endpoint: {self.proxy_endpoint}, partition: {self.partition}"
)
def run(self):
# batch stream must be executed serially, and reinit.
# TODO:0: remove lock to bss
rs_header = None
with PutBatchTask._partition_lock[self.tag]: # tag includes partition info in tag generation
L.trace(f"do_store start for tag={self.tag}, partition_id={self.partition._id}")
bss = _BatchStreamStatus.get_or_create(self.tag)
try:
broker = TransferService.get_or_create_broker(self.tag, write_signals=1)
iter_wait = 0
iter_timeout = int(CoreConfKeys.EGGROLL_CORE_FIFOBROKER_ITER_TIMEOUT_SEC.get())
batch = None
batch_get_interval = 0.1
with create_adapter(self.partition) as db, db.new_batch() as wb:
#for batch in broker:
while not broker.is_closable():
try:
batch = broker.get(block=True, timeout=batch_get_interval)
except queue.Empty as e:
iter_wait += batch_get_interval
if iter_wait > iter_timeout:
raise TimeoutError(f'timeout in PutBatchTask.run. tag={self.tag}, iter_timeout={iter_timeout}, iter_wait={iter_wait}')
else:
continue
except BrokerClosed as e:
continue
iter_wait = 0
rs_header = ErRollSiteHeader.from_proto_string(batch.header.ext)
batch_pairs = 0
if batch.data:
bin_data = ArrayByteBuffer(batch.data)
reader = PairBinReader(pair_buffer=bin_data, data=batch.data)
for k_bytes, v_bytes in reader.read_all():
wb.put(k_bytes, v_bytes)
batch_pairs += 1
bss.count_batch(rs_header, batch_pairs)
# TODO:0
bss._data_type = rs_header._data_type
if rs_header._stage == FINISH_STATUS:
bss.set_done(rs_header) # starting from 0
bss.check_finish()
# TransferService.remove_broker(tag) will be called in get_status phrase finished or exception got
except Exception as e:
L.exception(f'_run_put_batch error, tag={self.tag}, '
f'rs_key={rs_header.get_rs_key() if rs_header is not None else None}, rs_header={rs_header}')
raise e
finally:
TransferService.remove_broker(self.tag)
def __init__(self, options: dict = None):
if options is None:
options = {}
super().__init__(options)
er_partition = options['er_partition']
self.partition = er_partition
self.store_locator = er_partition._store_locator
self.partition_id = er_partition._id
self.namespace = self.store_locator._namespace
#_store_type = StoreTypes.ROLLPAIR_ROLLSITE
# self._store_locator = meta_pb2.StoreLocator(storeType=_store_type,
# namespace=self.namespace,
# name=self.store_locator._name,
# partitioner=self.store_locator._partitioner,
# serdes=self.store_locator._serdes,
# totalPartitions=self.store_locator._total_partitions)
self.roll_site_header_string = options.get('roll_site_header', None)
self.is_writable = False
if self.roll_site_header_string:
self.roll_site_header = ErRollSiteHeader.from_proto_string(
self.roll_site_header_string.encode(stringify_charset))
self.roll_site_header._options['partition_id'] = self.partition_id
self.proxy_endpoint = ErEndpoint.from_proto_string(
options['proxy_endpoint'].encode(stringify_charset))
self.obj_type = options['obj_type']
self.is_writable = True
L.trace(
f"writable RollSiteAdapter: {self.namespace}, partition_id={self.partition_id}. proxy_endpoint={self.proxy_endpoint}, partition={self.partition}"
)
self.unarycall_max_retry_cnt = int(
RollSiteConfKeys.EGGROLL_ROLLSITE_UNARYCALL_CLIENT_MAX_RETRY.
get_with(options))
self.push_max_retry_cnt = int(
RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_CLIENT_MAX_RETRY.get_with(
options))
self.push_overall_timeout = int(
RollSiteConfKeys.EGGROLL_ROLLSITE_OVERALL_TIMEOUT_SEC.get_with(
options))
self.push_completion_wait_timeout = int(
RollSiteConfKeys.EGGROLL_ROLLSITE_COMPLETION_WAIT_TIMEOUT_SEC.
get_with(options))
self.push_packet_interval_timeout = int(
RollSiteConfKeys.EGGROLL_ROLLSITE_PACKET_INTERVAL_TIMEOUT_SEC.
get_with(options))