def f_create_vocab(self, vocab_obj, train_reviews):
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
# train_reviews = train_df.review
max_line = self.m_max_line
line_i = 0
for review in train_reviews:
words = tokenizer.tokenize(review)
w2c.update(words)
if line_i > max_line:
break
line_i += 1
print("max line", max_line)
for w, c in w2c.items():
if c > self.m_min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
print("len(i2w)", len(i2w))
vocab_obj.f_set_vocab(w2i, i2w)
def create_vocab(self, hints, test_hints):
w2i = dict()
i2w = dict()
w2c = OrderedCounter()
special_tokens = [PAD_TOKEN, SOS_TOKEN, EOS_TOKEN, UNK_TOKEN]
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
for hint in hints:
hint_tokens = hint.split()
w2c.update(hint_tokens)
if test_hints is not None:
for hint in test_hints:
hint_tokens = hint.split()
w2c.update(hint_tokens)
# sort token so that different instantiations of the dataset is compatible
for w, c in sorted(list(w2c.items())):
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w) == len(w2c)+len(special_tokens)
vocab = dict(w2i=w2i, i2w=i2w, w2c=w2c)
self.vocab = vocab
logging.info('Created vocab with %d words.' % len(w2c))
def _create_vocab(self):
assert self.split == 'train', "Vocablurary can only be created for training file."
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
patients = np.load(self.raw_data_path)[()]
for patient in patients.keys():
for visit in patients[patient]:
w2c.update(visit)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." %len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file), 'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def create_vocab(self, hints, test_hints):
w2i = dict()
i2w = dict()
w2c = OrderedCounter()
special_tokens = [PAD_TOKEN, SOS_TOKEN, EOS_TOKEN, UNK_TOKEN]
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
for hint in hints:
hint_tokens = hint.split()
w2c.update(hint_tokens)
if test_hints is not None:
for hint in test_hints:
hint_tokens = hint.split()
w2c.update(hint_tokens)
for w, c in list(w2c.items()):
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
vocab = dict(w2i=w2i, i2w=i2w)
self.vocab = vocab
logging.info('Created vocab with %d words.' % len(w2c))
def __generate_features_dictionaries__(self):
for feature in self.getFeatureNames():
self.features_dicts[feature] = OrderedCounter()
for graph in self.graphs:
for edge in graph.get_edges():
self.__basic_features_dictionaries__(edge)
self.__add_basic_missing_tags__()
def create_vocab(self, data):
assert self.split == 'train', \
"Vocabulary can only be created for training file."
w2c = OrderedCounter()
w2i, i2w = dict(), dict()
special_tokens = [PAD_TOKEN, UNK_TOKEN, SOS_TOKEN, EOS_TOKEN]
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
for program in data:
tokens = program.split()
w2c.update(tokens)
for w, c in w2c.items():
if c > self.min_occ:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocabulary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
return vocab
def _create_vocab(self):
assert self.split == 'train', "Vocabulary can only be created for training file."
tokenizer = PunktSentenceTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
with open(self.raw_data_path, 'r') as file:
for i, line in enumerate(file):
words = tokenizer.tokenize(line)
w2c.update(words)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocabulary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self):
assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
data_folder = self.raw_data_path + "/" + self.split
line_num = 0
for filename in os.listdir(data_folder):
if "news.en" not in filename:
continue
if line_num > self.max_line:
break
full_filename = os.path.join(data_folder, filename)
print("file", full_filename)
file = open(full_filename, "r")
print("max line", self.max_line)
for i, line in enumerate(file):
words = tokenizer.tokenize(line)
w2c.update(words)
line_num += 1
if line_num > self.max_line:
break
print("line_num", line_num)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self):
assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
self.w2c = OrderedCounter()
self.w2i = dict()
self.i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
self.i2w[len(self.w2i)] = st
self.w2i[st] = len(self.w2i)
labels = ['0', '1']
for l in labels:
print("updating vocab with sentences of label {}".format(l))
file = open(self.raw_data_path + l, 'r')
num_lines = self.num_lines_0 if l=='0' else self.num_lines_1
for i, line in enumerate(tqdm(file, total=num_lines)):
if(i == num_lines):
break
words = tokenizer.tokenize(line)
if(len(words) > self.max_sequence_length):
continue
self.w2c.update(words)
file.close()
print("done creating w2c")
for w, c in tqdm(self.w2c.items()):
if c > self.min_occ and w not in special_tokens:
self.i2w[len(self.w2i)] = w
self.w2i[w] = len(self.w2i)
print("done creating w2i")
assert len(self.w2i) == len(self.i2w)
print("Vocablurary of %i keys created." % len(self.w2i))
vocab = dict(w2i=self.w2i, i2w=self.i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file), 'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
self.v_size = len(self.w2i)
def _create_combined_vocab(self):
# this function uses both snli + yelp to create vocab
assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
# first for yelp
with open(self.yelp_raw_data_path, 'r') as file:
for i, line in enumerate(tqdm(file, total=self.num_lines)):
if (i == self.num_lines):
break
words = tokenizer.tokenize(line)
w2c.update(words)
# now for snli
with open(self.snli_raw_data_path, 'r') as file:
for i, line in enumerate(tqdm(file, total=self.num_lines)):
if (i == self.num_lines):
break
words = tokenizer.tokenize(line)
w2c.update(words)
for w, c in tqdm(w2c.items()):
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocabulary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self, raw_data_file, **kwargs):
assert self.split == 'train', "Only for training data Vocablurary can be created."
print("Creating New Vocablurary.")
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
with open(raw_data_file) as file:
for i, line in enumerate(file):
line = self._preprocess(line)
question, answer = line.split('|||')
question = tokenizer.tokenize(question)
question = question[:self.max_utterance_length]
answer = tokenizer.tokenize(answer)
answer = answer[:self.max_utterance_length -
1] # sos or eos token will be added
words = question + answer
w2c.update(question + answer)
if i > 1000000:
break
for w, c in w2c.items():
if c > self.min_occ:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." % len(w2i))
vocab_file_path = os.path.join(self.root, self.vocab_file_name)
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(vocab_file_path, 'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def create_vocab(self, vocab_file):
self.vocab_file = vocab_file
df = pd.DataFrame()
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
preprocess_word = Preprocess_Word()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
with open(self.vocab_file, 'r') as file:
lines = []
for i, line in enumerate(file):
lines.append(line)
#line = rewrite_to_toklen(line)
#words = tokenizer.tokenize(line)
#words = [c for c in line]
words = preprocess_word.to_words(line)
w2c.update(words)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." % len(w2i))
self.w2i = w2i
self.i2w = i2w
self.dump()
if self.train_with_vocab:
df['url'] = lines
self.create_data(df)
elif self.train_file is not None:
df = pd.read_csv(self.train_file, names=['url'])
self.create_data(df)
def f_create_vocab(self, vocab_obj):
# assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
with open(self.m_raw_train_data_path, 'r') as file:
max_i = 0
for i, line in enumerate(file):
words = tokenizer.tokenize(line)
w2c.update(words)
max_i = i
if i > self.m_max_line:
break
print("max_i", max_i)
for w, c in w2c.items():
if c > self.m_min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.m_data_dir, self.m_vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
print("len(i2w)", len(i2w))
vocab_obj.f_set_vocab(w2i, i2w)
def _create_vocab(self):
assert self.split == 'train', "Vocabulary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
df = pandas.read_csv(self.raw_data_path)
for i in range(len(df)):
poem = df.iloc[i]["Poem"]
poem = poem.replace("\r\r\n", " <nl> ")
words = tokenizer.tokenize(poem)
# Filter out poems that don't have newlines
if words.count('<nl>') <= 2:
continue
w2c.update(words)
for w, c in w2c.items():
if c >= self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocabulary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self):
assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
#print("PATH: ", self.raw_data_path)
if self.rows > 0:
file = pd.read_csv(self.raw_data_path, nrows=self.rows)['text']
else:
file = pd.read_csv(self.raw_data_path)['text']
#print("Data size: ", file.shape, )
file = file.dropna(axis=0)
for i, line in enumerate(file):
#if i == 27054:
# continue
words = tokenizer.tokenize(line)
w2c.update(words)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
#print("Vocabulary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.vocab_directory, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self, vocab_obj, train_reviews):
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
# train_reviews = train_df.review
max_line = self.m_max_line
line_i = 0
for review in train_reviews:
words = tokenizer.tokenize(review)
w2c.update(words)
if line_i > max_line:
break
line_i += 1
print("max line", max_line)
for w, c in w2c.items():
if c > self.m_min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
# print("vocabulary of %i keys created"%len(w2i))
# vocab = dict(w2i=w2i, i2w=i2w)
# with io.open(os.path.join(self.m_data_dir, self.m_vocab_file), 'wb') as vocab_file:
# data = json.dumps(vocab, ensure_ascii=False)
# vocab_file.write(data.encode('utf8', 'replace'))
print("len(i2w)", len(i2w))
vocab_obj.f_set_vocab(w2i, i2w)
def _create_vocab(self, dataset_raw_file, **kwargs):
assert self.split == 'train', "Vocablurary can only be created from training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
# add speical tokens to vocab
special_tokens = ['<pad>', '<sos>', '<eos>', '<unk>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
with open(dataset_raw_file, 'r') as file:
# read data and count token occurences
for line in file.readlines():
tokens = tokenizer.tokenize(line)
w2c.update(tokens)
# create vocab with
for w, c in w2c.items():
if c > self.min_occ:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
vocab = dict(w2i=w2i, i2w=i2w)
# save vocab to file
vocab_file_path = os.path.join(self.root, 'vocab.json')
with io.open(vocab_file_path, 'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
print(
"Vocablurary created with %i tokens. Minimum occurence criterion = %i."
% (len(w2i), self.min_occ))
self._load_vocab()
def _create_vocab(self):
assert self.split == 'train', "Vocablurary can only be created for training file."
w2c = OrderedCounter()
w2i = dict()
i2w = dict()
special_tokens = ['<pad>', '<unk>', '<sos>', '<eos>']
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
# NOTE: vocab中 で特殊トークンはConditionEncoderでも横断的に現れそうなので、
# そのidは統一しておきたい
assert w2i['<pad>'] == PAD_INDEX
assert w2i['<unk>'] == UNK_INDEX
assert w2i['<sos>'] == SOS_INDEX
assert w2i['<eos>'] == EOS_INDEX
with open(self.raw_data_path, 'r') as file:
for i, line in enumerate(file):
words = self.tokenize(line)
w2c.update(words)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
print("Vocablurary of %i keys created." % len(w2i))
vocab = dict(w2i=w2i, i2w=i2w)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab()
def _create_vocab(self, bert):
assert self.split == 'train', "Vocablurary can only be created for training file."
tokenizer = TweetTokenizer(preserve_case=False)
w2c = OrderedCounter()
a2c = OrderedCounter()
w2i = dict()
i2w = dict()
a2i = dict()
i2a = dict()
if bert:
self.pad = '[PAD]'
self.unk = '[UNK]'
self.sos = '[CLS]'
self.eos = '[SEP]'
else:
self.pad = '[PAD]'
self.unk = '[UNK]'
self.sos = '[SOS]'
self.eos = '[EOS]'
special_tokens = [self.pad, self.unk, self.sos, self.eos]
for st in special_tokens:
i2w[len(w2i)] = st
w2i[st] = len(w2i)
i2a[len(a2i)] = st
a2i[st] = len(a2i)
with open(self.raw_definition_path, 'r') as file:
for i, line in enumerate(file):
words = tokenizer.tokenize(line)
w2c.update(words)
for w, c in w2c.items():
if c > self.min_occ and w not in special_tokens:
i2w[len(w2i)] = w
w2i[w] = len(w2i)
assert len(w2i) == len(i2w)
with open(self.raw_word_path, 'r') as file:
for i, line in enumerate(file):
words = list(line.strip())
a2c.update(words)
for w, c in a2c.items():
if c > self.min_occ and w not in special_tokens:
i2a[len(a2i)] = w
a2i[w] = len(a2i)
assert len(a2i) == len(i2a)
print("Vocabulary of %i keys created." % len(w2i))
print("Alphabet of %i keys created." % len(a2i))
vocab = dict(w2i=w2i, i2w=i2w, a2i=a2i, i2a=i2a)
with io.open(os.path.join(self.data_dir, self.vocab_file),
'wb') as vocab_file:
data = json.dumps(vocab, ensure_ascii=False)
vocab_file.write(data.encode('utf8', 'replace'))
self._load_vocab(bert)