def _init_dataset_maker(self, load=False):
if not load:
DatasetMaker.generate_mapping(self.train_data)
DatasetMaker.save_mapping(self.map_file, self.vocabulary_file)
else:
DatasetMaker.load_mapping(self.map_file)
DatasetMaker.save_mapping(self.map_file, self.vocabulary_file)
FLAGS.char_num = len(DatasetMaker.char_to_id)
FLAGS.tag_num = len(DatasetMaker.tag_to_id)
def _init_dataset_maker(self, load=False):
if not load:
DatasetMaker.generate_mapping(self.train_data)
if self.is_chief:
DatasetMaker.save_mapping(self.map_file, self.vocabulary_file)
else:
DatasetMaker.load_mapping(self.map_file)
if self.is_chief:
DatasetMaker.save_mapping(self.map_file, self.vocabulary_file)
FLAGS.char_num = len(DatasetMaker.char_to_id)
#FLAGS.gram2_num = len(DatasetMaker.gram2_to_id)
#FLAGS.gram3_num = len(DatasetMaker.gram3_to_id)
FLAGS.label_num = len(DatasetMaker.label_to_id)