def setUp(self):
self.etcd = etcd_client.EtcdClient('test_cluster', 'localhost:2379',
'fedlearner', True)
data_source = common_pb.DataSource()
data_source.data_source_meta.name = "milestone-x"
data_source.data_source_meta.partition_num = 1
data_source.output_base_dir = "./ds_output"
self.etcd.delete_prefix(common.data_source_etcd_base_dir(data_source.data_source_meta.name))
self.data_source = data_source
self.example_id_dump_options = dj_pb.ExampleIdDumpOptions(
example_id_dump_interval=-1,
example_id_dump_threshold=1024
)
if gfile.Exists(self.data_source.output_base_dir):
gfile.DeleteRecursively(self.data_source.output_base_dir)
self.partition_dir = os.path.join(common.data_source_example_dumped_dir(self.data_source), common.partition_repr(0))
gfile.MakeDirs(self.partition_dir)
def _example_dumped_dir(self):
return os.path.join(data_source_example_dumped_dir(self._data_source),
partition_repr(self._partition_id))