def tearDown(self) -> None:
dir = os.path.join(self.args.data_config.data_dir,
self.args.data_config.data_prep_dir)
if utils.exists_dir(dir):
shutil.rmtree(dir)
dir = os.path.join(self.args.data_config.entity_dir,
self.args.data_config.entity_prep_dir)
if utils.exists_dir(dir):
shutil.rmtree(dir)
if os.path.exists(self.temp_file_name):
os.remove(self.temp_file_name)
def tearDown(self) -> None:
dir = os.path.join(self.args.data_config.data_dir,
self.args.data_config.data_prep_dir)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join(self.args.data_config.entity_dir,
self.args.data_config.entity_prep_dir)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join("test/temp_log")
if os.path.exists(dir):
shutil.rmtree(dir, ignore_errors=True)
def tearDown(self) -> None:
dir = os.path.join(
self.config["data_config"]["entity_dir"],
self.config["data_config"]["entity_prep_dir"],
)
if utils.exists_dir(dir):
shutil.rmtree(dir)
def tearDown(self) -> None:
if os.path.exists(self.type_file):
os.remove(self.type_file)
if os.path.exists(self.type_vocab_file):
os.remove(self.type_vocab_file)
if os.path.exists(self.regularization_csv):
os.remove(self.regularization_csv)
dir = os.path.join(
self.args.data_config.data_dir, self.args.data_config.data_prep_dir
)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join(
self.args.data_config.entity_dir, self.args.data_config.entity_prep_dir
)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join("test/temp_log")
if os.path.exists(dir):
shutil.rmtree(dir, ignore_errors=True)
def tearDown(self) -> None:
if os.path.exists(self.regularization_csv):
os.remove(self.regularization_csv)
if os.path.exists(self.qid2topkeid):
os.remove(self.qid2topkeid)
if os.path.exists(self.static_emb):
os.remove(self.static_emb)
dir = os.path.join(
self.args.data_config.data_dir, self.args.data_config.data_prep_dir
)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join(
self.args.data_config.entity_dir, self.args.data_config.entity_prep_dir
)
if utils.exists_dir(dir):
shutil.rmtree(dir, ignore_errors=True)
dir = os.path.join("test/temp_log")
if os.path.exists(dir):
shutil.rmtree(dir, ignore_errors=True)
def __init__(self, args, is_writer, distributed):
super(CustomWordSymbols, self).__init__(args, is_writer, distributed)
self.is_bert = False
if args.overwrite_preprocessed_data:
# TODO: this still isn't thread safe for prep
path_pt = os.path.join(args.emb_dir, args.word_embedding.custom_vocab_embedding_file) + ".pt"
if utils.exists_dir(path_pt) and is_writer:
os.remove(path_pt)
if distributed:
dist.barrier()
word_embeddings = torchtext.vocab.Vectors(
args.word_embedding.custom_vocab_embedding_file, cache=args.emb_dir)
self.vocab = word_embeddings.stoi
self.word_embedding_dim = word_embeddings.vectors.shape[1]
self.num_words = word_embeddings.vectors.shape[0]
del word_embeddings