def _new_writer(self):
assert self._writer is None
if self._options.output_builder == 'TF_RECORD':
self._writer = tf.io.TFRecordWriter(self._tmp_fpath)
else:
assert self._options.output_builder == 'CSV_DICT'
self._writer = CsvDictWriter(self._tmp_fpath)
def _new_writer(self):
assert self._writer is None
fname = "{:04}-{:08}.rd".format(
self._options.partitioner_rank_id,
self._process_index
)
fpath = os.path.join(self._output_dir, fname)
self._output_fpaths.append(fpath)
if self._options.output_builder == 'TF_RECORD':
self._writer = tf.io.TFRecordWriter(fpath)
else:
assert self._options.output_builder == 'CSV_DICT'
self._writer = CsvDictWriter(fpath)
self._dumped_item = 0
def _make_data_block_writer(self, fpath):
return CsvDictWriter(fpath)
def __init__(self, options, fpath):
super(CSVDictBuilder, self).__init__(options, fpath)
self._writer = CsvDictWriter(fpath)