def start_create_data_block(self, partition_id):
with self._lock:
dump_manager = self._data_block_dump_manager
if (dump_manager is not None
and dump_manager.get_partition_id() != partition_id):
raise RuntimeError("partition {} is not finished".format(
dump_manager.get_partition_id()))
if dump_manager is None:
self._data_block_dump_manager = DataBlockDumperManager(
self._etcd, self._data_source, partition_id)
dump_manager = self._data_block_dump_manager
next_index = dump_manager.get_next_data_block_index()
return next_index
class ImplContext(TransmitFollower.ImplContext):
def __init__(self, etcd, data_source, partition_id, raw_data_options,
data_block_builder_options):
super(ExampleJoinFollower.ImplContext, self).__init__(partition_id)
self.data_block_dumper_manager = DataBlockDumperManager(
etcd, data_source, partition_id, raw_data_options,
data_block_builder_options)
def get_next_index(self):
return self.data_block_dumper_manager.get_next_data_block_index()
def get_dumped_index(self):
return self.data_block_dumper_manager.get_dumped_data_block_index()
def add_synced_content(self, sync_ctnt):
return self.data_block_dumper_manager.add_synced_data_block_meta(
sync_ctnt.data_block_meta)
def finish_sync_content(self):
self.data_block_dumper_manager.finish_sync_data_block_meta()
def need_dump(self):
return self.data_block_dumper_manager.need_dump()
def make_dumper(self):
return self.data_block_dumper_manager.make_data_block_dumper()
def is_sync_content_finished(self):
dumper_manager = self.data_block_dumper_manager
return dumper_manager.is_synced_data_block_meta_finished()
class ExampleJoinFollower(object):
def __init__(self, etcd, data_source):
self._lock = threading.Lock()
self._etcd = etcd
self._data_source = data_source
self._data_block_dump_manager = None
self._data_block_dump_worker = None
self._started = False
def start_create_data_block(self, partition_id):
with self._lock:
dump_manager = self._data_block_dump_manager
if (dump_manager is not None
and dump_manager.get_partition_id() != partition_id):
raise RuntimeError("partition {} is not finished".format(
dump_manager.get_partition_id()))
if dump_manager is None:
self._data_block_dump_manager = DataBlockDumperManager(
self._etcd, self._data_source, partition_id)
dump_manager = self._data_block_dump_manager
next_index = dump_manager.get_next_data_block_index()
return next_index
def add_synced_data_block_meta(self, meta):
with self._lock:
self._check_status(meta.partition_id)
manager = self._data_block_dump_manager
return manager.append_synced_data_block_meta(meta)
def finish_sync_data_block_meta(self, partition_id):
with self._lock:
self._check_status(partition_id)
self._data_block_dump_manager.finish_sync_data_block_meta()
return not self._data_block_dump_manager.need_dump()
def get_processing_partition_id(self):
with self._lock:
if self._data_block_dump_manager is None:
return None
return self._data_block_dump_manager.get_partition_id()
def reset_dump_partition(self):
with self._lock:
if self._data_block_dump_manager is None:
return
dump_manager = self._data_block_dump_manager
partition_id = dump_manager.get_partition_id()
self._check_status(partition_id)
if (not dump_manager.data_block_meta_sync_finished()
or dump_manager.need_dump()):
raise RuntimeError(
"partition {} is dumpping".format(partition_id))
self._data_block_dump_manager = None
def start_dump_worker(self):
with self._lock:
if not self._started:
assert self._data_block_dump_worker is None
self._data_block_dump_worker = RoutineWorker(
'data_block_dumper', self._dump_data_block_fn,
self._dump_data_block_cond, 1)
self._data_block_dump_worker.start_routine()
self._started = True
def stop_dump_worker(self):
dumper = None
with self._lock:
if self._data_block_dump_worker is not None:
dumper = self._data_block_dump_worker
self._data_block_dump_worker = None
if dumper is not None:
dumper.stop_routine()
def _check_status(self, partition_id):
if self._data_block_dump_manager is None:
raise RuntimeError("no partition is processing")
ptn_id = self._data_block_dump_manager.get_partition_id()
if partition_id != ptn_id:
raise RuntimeError("partition id mismatch {} != {}".format(
partition_id, ptn_id))
def _dump_data_block_fn(self):
dump_manager = self._data_block_dump_manager
assert dump_manager is not None
if dump_manager.need_dump():
dump_manager.dump_data_blocks()
def _dump_data_block_cond(self):
with self._lock:
return (self._data_block_dump_manager is not None
and self._data_block_dump_manager.need_dump())
def __init__(self, etcd, data_source, partition_id, raw_data_options,
data_block_builder_options):
super(ExampleJoinFollower.ImplContext, self).__init__(partition_id)
self.data_block_dumper_manager = DataBlockDumperManager(
etcd, data_source, partition_id, raw_data_options,
data_block_builder_options)