def setUp(self): self.etcd = etcd_client.EtcdClient('test_cluster', 'localhost:2379', 'fedlearner', True) data_source = common_pb.DataSource() data_source.data_source_meta.name = "milestone-x" data_source.data_source_meta.partition_num = 1 data_source.output_base_dir = "./ds_output" self.etcd.delete_prefix(common.data_source_etcd_base_dir(data_source.data_source_meta.name)) self.data_source = data_source self.example_id_dump_options = dj_pb.ExampleIdDumpOptions( example_id_dump_interval=-1, example_id_dump_threshold=1024 ) if gfile.Exists(self.data_source.output_base_dir): gfile.DeleteRecursively(self.data_source.output_base_dir) self.partition_dir = os.path.join(common.data_source_example_dumped_dir(self.data_source), common.partition_repr(0)) gfile.MakeDirs(self.partition_dir)
def _example_dumped_dir(self): return os.path.join(data_source_example_dumped_dir(self._data_source), partition_repr(self._partition_id))