def getData_new_method(USE_BPE):
if USE_BPE == False:
g_tok = get_tokenizer('spacy', language='de')
e_tok = get_tokenizer('spacy', language='en')
return getData2(g_tok, e_tok, USE_BPE)
else:
if config.BPE_FROM_PICKLE:
pkl_file = open('BPE/data.pkl', 'rb')
data1 = pickle.load(pkl_file)
sp_gec = data1["sp_gec_orig"]
else:
sp_gec = load_sp_model(config.BPE_PATH)
return getData2(sp_gec, sp_gec, USE_BPE)
def getData_new_method():
g_tok = get_tokenizer('spacy', language='en')
e_tok = get_tokenizer('spacy', language='de')
return getData2(g_tok, e_tok)