def __init__(self, etcd, raw_data_options, mock_data_source_name,
raw_data_sub_dir):
mock_data_source = common_pb.DataSource(
state=common_pb.DataSourceState.Processing,
data_source_meta=common_pb.DataSourceMeta(
name=mock_data_source_name,
partition_num=1,
raw_data_sub_dir=raw_data_sub_dir))
self._mock_rd_manifest_manager = RawDataManifestManager(
etcd, mock_data_source)
super(EtcdBasedMockRawDataVisitor,
self).__init__(etcd, mock_data_source, 0, raw_data_options)
def __init__(self, kvstore, raw_data_options, mock_data_source_name,
raw_data_sub_dir, partition_id):
mock_data_source = common_pb.DataSource(
state=common_pb.DataSourceState.Processing,
raw_data_sub_dir=raw_data_sub_dir,
data_source_meta=common_pb.DataSourceMeta(
name=mock_data_source_name, partition_num=partition_id + 1))
self._mock_rd_manifest_manager = RawDataManifestManager(
kvstore, mock_data_source, False)
self._partition_id = partition_id
super(DBBasedMockRawDataVisitor,
self).__init__(kvstore, mock_data_source, partition_id,
raw_data_options)
def __init__(self, etcd, raw_data_options, mock_data_source_name,
input_fpaths):
mock_data_source = common_pb.DataSource(
state=common_pb.DataSourceState.Processing,
data_source_meta=common_pb.DataSourceMeta(
name=mock_data_source_name, partition_num=1))
mock_rd_manifest_manager = RawDataManifestManager(
etcd, mock_data_source)
manifest = mock_rd_manifest_manager.get_manifest(0)
if not manifest.finished:
metas = []
for fpath in input_fpaths:
metas.append(dj_pb.RawDataMeta(file_path=fpath,
start_index=-1))
mock_rd_manifest_manager.add_raw_data(0, metas, True)
mock_rd_manifest_manager.finish_raw_data(0)
super(MockRawDataVisitor, self).__init__(etcd, mock_data_source, 0,
raw_data_options)
def __init__(self, peer_client, data_source_name, kvstore, batch_mode):
self._lock = threading.Lock()
self._peer_client = peer_client
self._data_source_name = data_source_name
self._kvstore = kvstore
self._batch_mode = batch_mode
self._init_fsm_action()
self._data_source = None
self._sync_data_source()
self._reset_batch_mode()
self._raw_data_manifest_manager = RawDataManifestManager(
kvstore, self._data_source, batch_mode)
self._data_source_meta = self._data_source.data_source_meta
if self._data_source.role == common_pb.FLRole.Leader:
self._role_repr = "leader"
else:
self._role_repr = "follower"
self._fsm_worker = None
self._started = False
def __init__(self, peer_client, data_source_name, etcd):
self._lock = threading.Lock()
self._peer_client = peer_client
self._data_source_name = data_source_name
self._etcd = etcd
self._init_fsm_action()
self._data_source = None
self._sync_data_source()
assert self._data_source is not None, \
"data source must not None if sync data source success"
self._raw_data_manifest_manager = RawDataManifestManager(
etcd, self._data_source
)
self._data_source_meta = self._data_source.data_source_meta
if self._data_source.role == common_pb.FLRole.Leader:
self._role_repr = "leader"
else:
self._role_repr = "follower"
self._fsm_worker = None
self._started = False