def _get_output_writer(self):
if self._writer is None:
self._writer = create_output_writer(
self._options.writer_options,
self._tmp_fpath
)
return self._writer
def __init__(self, dirname, data_source_name, partition_id,
data_block_index, write_options, max_example_num=None):
self._data_source_name = data_source_name
self._partition_id = partition_id
self._max_example_num = max_example_num
self._dirname = dirname
self._tmp_fpath = self._get_tmp_fpath()
self._writer = create_output_writer(write_options, self._tmp_fpath)
self._data_block_meta = dj_pb.DataBlockMeta()
self._data_block_meta.partition_id = partition_id
self._data_block_meta.data_block_index = data_block_index
self._data_block_meta.follower_restart_index = 0
self._example_num = 0
self._data_block_manager = None
self._example_ids_size = 0
self._metrics_tags = {'ds_name': self._data_source_name,
'partition': partition_id}
def _get_output_writer(self, fpath):
if self._writer is None:
self._writer = create_output_writer(self._writer_options,
fpath)
return self._writer
def _get_output_writer(self):
if self._writer is None:
self._tmp_fpath = common.gen_tmp_fpath(self._merged_dir)
self._writer = create_output_writer(self._writer_options,
self._tmp_fpath)
return self._writer
