def __init__(self, options: dict = None): if options is None: options = {} super().__init__(options) er_partition = options['er_partition'] self.partition = er_partition self.store_locator = er_partition._store_locator self.partition_id = er_partition._id self.namespace = self.store_locator._namespace #_store_type = StoreTypes.ROLLPAIR_ROLLSITE # self._store_locator = meta_pb2.StoreLocator(storeType=_store_type, # namespace=self.namespace, # name=self.store_locator._name, # partitioner=self.store_locator._partitioner, # serdes=self.store_locator._serdes, # totalPartitions=self.store_locator._total_partitions) self.roll_site_header_string = options.get('roll_site_header', None) self.is_writable = False if self.roll_site_header_string: self.roll_site_header = ErRollSiteHeader.from_proto_string( self.roll_site_header_string.encode(stringify_charset)) self.proxy_endpoint = ErEndpoint.from_proto_string( options['proxy_endpoint'].encode(stringify_charset)) self.obj_type = options['obj_type'] self.is_writable = True L.info( f"writable RollSiteAdapter: {self.namespace}, {self.partition_id}. proxy_endpoint: {self.proxy_endpoint}, partition: {self.partition}" )
def run(self): # batch stream must be executed serially, and reinit. # TODO:0: remove lock to bss rs_header = None with PutBatchTask._partition_lock[self.tag]: # tag includes partition info in tag generation L.trace(f"do_store start for tag={self.tag}, partition_id={self.partition._id}") bss = _BatchStreamStatus.get_or_create(self.tag) try: broker = TransferService.get_or_create_broker(self.tag, write_signals=1) iter_wait = 0 iter_timeout = int(CoreConfKeys.EGGROLL_CORE_FIFOBROKER_ITER_TIMEOUT_SEC.get()) batch = None batch_get_interval = 0.1 with create_adapter(self.partition) as db, db.new_batch() as wb: #for batch in broker: while not broker.is_closable(): try: batch = broker.get(block=True, timeout=batch_get_interval) except queue.Empty as e: iter_wait += batch_get_interval if iter_wait > iter_timeout: raise TimeoutError(f'timeout in PutBatchTask.run. tag={self.tag}, iter_timeout={iter_timeout}, iter_wait={iter_wait}') else: continue except BrokerClosed as e: continue iter_wait = 0 rs_header = ErRollSiteHeader.from_proto_string(batch.header.ext) batch_pairs = 0 if batch.data: bin_data = ArrayByteBuffer(batch.data) reader = PairBinReader(pair_buffer=bin_data, data=batch.data) for k_bytes, v_bytes in reader.read_all(): wb.put(k_bytes, v_bytes) batch_pairs += 1 bss.count_batch(rs_header, batch_pairs) # TODO:0 bss._data_type = rs_header._data_type if rs_header._stage == FINISH_STATUS: bss.set_done(rs_header) # starting from 0 bss.check_finish() # TransferService.remove_broker(tag) will be called in get_status phrase finished or exception got except Exception as e: L.exception(f'_run_put_batch error, tag={self.tag}, ' f'rs_key={rs_header.get_rs_key() if rs_header is not None else None}, rs_header={rs_header}') raise e finally: TransferService.remove_broker(self.tag)
def __init__(self, options: dict = None): if options is None: options = {} super().__init__(options) er_partition = options['er_partition'] self.partition = er_partition self.store_locator = er_partition._store_locator self.partition_id = er_partition._id self.namespace = self.store_locator._namespace #_store_type = StoreTypes.ROLLPAIR_ROLLSITE # self._store_locator = meta_pb2.StoreLocator(storeType=_store_type, # namespace=self.namespace, # name=self.store_locator._name, # partitioner=self.store_locator._partitioner, # serdes=self.store_locator._serdes, # totalPartitions=self.store_locator._total_partitions) self.roll_site_header_string = options.get('roll_site_header', None) self.is_writable = False if self.roll_site_header_string: self.roll_site_header = ErRollSiteHeader.from_proto_string( self.roll_site_header_string.encode(stringify_charset)) self.roll_site_header._options['partition_id'] = self.partition_id self.proxy_endpoint = ErEndpoint.from_proto_string( options['proxy_endpoint'].encode(stringify_charset)) self.obj_type = options['obj_type'] self.is_writable = True L.trace( f"writable RollSiteAdapter: {self.namespace}, partition_id={self.partition_id}. proxy_endpoint={self.proxy_endpoint}, partition={self.partition}" ) self.unarycall_max_retry_cnt = int( RollSiteConfKeys.EGGROLL_ROLLSITE_UNARYCALL_CLIENT_MAX_RETRY. get_with(options)) self.push_max_retry_cnt = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PUSH_CLIENT_MAX_RETRY.get_with( options)) self.push_overall_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_OVERALL_TIMEOUT_SEC.get_with( options)) self.push_completion_wait_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_COMPLETION_WAIT_TIMEOUT_SEC. get_with(options)) self.push_packet_interval_timeout = int( RollSiteConfKeys.EGGROLL_ROLLSITE_PACKET_INTERVAL_TIMEOUT_SEC. get_with(options))