def _load_data(df, use_preprocessdata=False, save_path=None):
"""Load preprocess task speccific data.
Args:
df (DataFrame): target pandas DataFrame object.
use_preprocessdata (bool): Wheter or not to use local preprocess file loading
save_path (str): local preprocess file path
Return:
data (List[tuple]): words and indexes tuple list. Tulpe foramt is (sentence_words, [Pronnoun, A, B])
"""
if use_preprocessdata:
try:
with open(save_path, 'rb') as f:
data = pickle.load(f)
except: # noqa
use_preprocessdata = False
if not use_preprocessdata:
data = []
for i in range(len(df)):
words, pronnoun_index = utils.charpos_to_word_index(
df['Text'][i], df['Pronoun-offset'][i],
df['Pronoun'][i].split()[0])
_, A_index = utils.charpos_to_word_index(df['Text'][i],
df['A-offset'][i],
df['A'][i].split()[0],
words=words)
_, B_index = utils.charpos_to_word_index(df['Text'][i],
df['B-offset'][i],
df['B'][i].split()[0],
words=words)
data.append((words, [pronnoun_index, A_index, B_index]))
with open(save_path, 'wb') as f:
pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
print("Data Loaded")
return data
def test_charpos_to_word_index_use_words():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
191, 'Cheryl')
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274,
'her',
words=words)
eq_(words[index].text, 'her')
def test_get_scope_sentence():
words, index1 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
words, index2 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
207, 'Pauline')
sentence = gpt2_estimator._get_scope_sentence(words,
np.array([index1, index2]))
eq_(
"Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
sentence)
def test_get_same_sentence_features():
words, index1 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
words, index2 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
191, 'Cheryl')
words, index3 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
207, 'Pauline')
feature_list = stanfordnlp_model._get_same_sentence_features(
words, [index1, index2, index3])
eq_(feature_list, [0, 0, 1])
def test_calcurate_likelihood():
words, index1 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
words, index2 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
191, 'Cheryl')
words, index3 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
207, 'Pauline')
predicts = gpt2_model.calcurate_likelihood(
words, np.array([index1, index2, index3]))
ok_(predicts[0] > 0)
ok_(predicts[1] == 0)
ok_(predicts[2] == 0)
def test_get_before_pronounce_sentence_case_false():
words, index1 = utils.charpos_to_word_index(
"Her initial ambition was to become a ``cultured'' private secretary working in the highest cultural circles. A month later, the Michigan League building opened, serving co-eds as the Michigan Union served men. Hartwig began by doing secretarial work there, when Dr. Margaret Bell, the head of women's physical education, spotted her.",
329, 'her')
is_possessive = gpt2_estimator._check_pronounce_is_possessive(
words, index1)
eq_(False, is_possessive)
def test_get_before_pronounce_sentence_case_true():
words, index1 = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
is_possessive = gpt2_estimator._check_pronounce_is_possessive(
words, index1)
eq_(True, is_possessive)
def test_vectorise_bag_of_pos_with_dependency_with_grandchild():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
191, 'Cheryl')
vectors = stanfordnlp_model._vectorise_bag_of_pos_with_dependency(
words, [index, index])
eq_(28, np.sum(vectors))
def test_get_bag_of_pos_with_dependency_with_child():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
207, 'Pauline')
poss = stanfordnlp_model._get_bag_of_pos_with_dependency(words, index)
eq_(3, len(poss))
eq_(['NN_governor', 'NNP_ancestor', 'POS_child'], poss)
def test_vectorise_bag_of_pos_with_position():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
vectors = stanfordnlp_model._vectorise_bag_of_pos_with_position(
words, [index, index], 3, targets=["a", "a b"])
eq_(7 * 2, np.sum(vectors))
eq_((2 * 36 * (stanfordnlp_model.DEFAULT_WINDOW_SIZE * 2 + 1), ),
vectors.shape)
def test_get_bag_of_pos_with_position():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
poss = stanfordnlp_model._get_bag_of_pos_with_position(words, index, 5)
eq_(11, len(poss))
eq_([
stanfordnlp_model._get_word_feature(w) + '_' + str(i - 5)
for i, w in enumerate(words[index - 5:index + 6])
], poss)
def test_get_bag_of_pos_ngram():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
poss = stanfordnlp_model._get_bag_of_pos_ngram(words, index, 5, 2)
eq_(11, len(poss))
eq_([
words[i].pos.replace('$', '') + "_" +
words[i + 1].pos.replace('$', '') for i in range(index - 5, index + 6)
], poss)
def test_charpos_to_word_index_case_error_pattern_1():
words, index = utils.charpos_to_word_index(
"Geller is reckoned to have been among the best ten players in the world for around twenty years. He was awarded the International Master title in 1951, and the International Grandmaster title the following year. Geller played in 23 USSR Chess Championship s, a record equalled by Mark Taimanov, achieving good results in many. He won in 1955 at Moscow (URS-ch22) when, despite losing five games, he finished with 12/19, then defeated Smyslov in the playoff match by the score of +1 =6.",
327, 'He')
eq_(words[index].text, 'He')
def test_get_bag_of_pos_case_start_0():
words, index = utils.charpos_to_word_index(
"Zoe Telford -- played the police officer girlfriend of Simon, Maggie. Dumped by Simon in the final episode of series 1, after he slept with Jenny, and is not seen again. Phoebe Thomas played Cheryl Cassidy, Pauline's friend and also a year 11 pupil in Simon's class. Dumped her boyfriend following Simon's advice after he wouldn't have sex with her but later realised this was due to him catching crabs off her friend Pauline.",
274, 'her')
eq_(11, len(stanfordnlp_model._get_bag_of_pos(words, 0, 5)))
