def __init__(self, etcd, data_source, partition_id, options):
self._lock = threading.Lock()
self._data_source = data_source
self._partition_id = partition_id
self._data_block_manager = DataBlockManager(data_source, partition_id)
self._raw_data_visitor = RawDataVisitor(etcd, data_source,
partition_id, options)
self._next_data_block_index = (
self._data_block_manager.get_dumped_data_block_num())
self._fly_data_block_meta = []
self._stale_with_dfs = False
self._synced_data_block_meta_finished = False
def __init__(self, etcd, data_source, partition_id, options):
self._data_source = data_source
self._partition_id = partition_id
self._leader_visitor = ExampleIdVisitor(
ExampleIdManager(data_source, partition_id))
self._follower_visitor = RawDataVisitor(etcd, data_source,
partition_id, options)
self._data_block_manager = DataBlockManager(data_source, partition_id)
self._data_block_builder = None
self._stale_with_dfs = False
self._follower_restart_index = 0
self._sync_state()
def __init__(self, etcd, data_source, partition_id, raw_data_options):
self._lock = threading.Lock()
self._data_source = data_source
self._partition_id = partition_id
self._data_block_manager = \
DataBlockManager(data_source, partition_id)
self._raw_data_visitor = \
RawDataVisitor(etcd, data_source,
partition_id, raw_data_options)
self._next_data_block_index = \
self._data_block_manager.get_dumped_data_block_count()
self._fly_data_block_meta = []
self._state_stale = False
self._synced_data_block_meta_finished = False
def __init__(self, etcd, data_source, partition_id,
raw_data_options, data_block_builder_options):
self._lock = threading.Lock()
self._data_source = data_source
self._partition_id = partition_id
self._data_block_manager = \
DataBlockManager(data_source, partition_id)
self._raw_data_visitor = \
RawDataVisitor(etcd, data_source,
partition_id, raw_data_options)
self._data_block_builder_options = data_block_builder_options
self._next_data_block_index = \
self._data_block_manager.get_dumped_data_block_count()
self._fly_data_block_meta = []
self._state_stale = False
self._synced_data_block_meta_finished = False
ds_name = self._data_source.data_source_meta.name
self._metrics_tags = {'data_source_name': ds_name,
'partiton': self._partition_id}
def __init__(self, example_joiner_options, raw_data_options,
data_block_builder_options, kvstore, data_source,
partition_id):
self._lock = threading.Lock()
self._example_joiner_options = example_joiner_options
self._raw_data_options = raw_data_options
self._data_source = data_source
self._partition_id = partition_id
self._leader_visitor = \
ExampleIdVisitor(kvstore, self._data_source, self._partition_id)
self._follower_visitor = \
RawDataVisitor(kvstore, self._data_source,
self._partition_id, raw_data_options)
self._data_block_manager = \
DataBlockManager(self._data_source, self._partition_id)
meta = self._data_block_manager.get_lastest_data_block_meta()
if meta is None:
self._joiner_stats = JoinerStats(0, -1, -1)
else:
stats_info = meta.joiner_stats_info
self._joiner_stats = JoinerStats(stats_info.stats_cum_join_num,
stats_info.leader_stats_index,
stats_info.follower_stats_index)
self._data_block_builder_options = data_block_builder_options
self._data_block_builder = None
self._state_stale = False
self._follower_restart_index = 0
self._sync_example_id_finished = False
self._raw_data_finished = False
self._join_finished = False
ds_name = self._data_source.data_source_meta.name
self._metrics_tags = {
'data_source_name': ds_name,
'partition': partition_id,
'joiner_name': self.name()
}
self._optional_stats = OptionalStats(raw_data_options,
self._metrics_tags)
self._latest_dump_timestamp = time.time()
self._sync_state()
def __init__(self, example_joiner_options, raw_data_options, etcd,
data_source, partition_id):
self._lock = threading.Lock()
self._example_joiner_options = example_joiner_options
self._raw_data_options = raw_data_options
self._data_source = data_source
self._partition_id = partition_id
self._leader_visitor = \
ExampleIdVisitor(etcd, self._data_source, self._partition_id)
self._follower_visitor = \
RawDataVisitor(etcd, self._data_source,
self._partition_id, raw_data_options)
self._data_block_manager = \
DataBlockManager(self._data_source, self._partition_id)
self._data_block_builder = None
self._state_stale = False
self._follower_restart_index = 0
self._sync_example_id_finished = False
self._raw_data_finished = False
self._join_finished = False
self._latest_dump_timestamp = time.time()
self._sync_state()