def test_shouldContainMethodWork(self):
o_text = ['I am fine, how about you?', 'One needs to hydrate regularly.']
d_text = 'I am fine, how about you? this weather is really hot and humid. ' \
'One needs to hydrate regularly to survive.'
d_sent_token = util.sent_tokenize(d_text)
sent_tokens = util.sent_tokenize(o_text[0])
for sent_token in sent_tokens:
self.assertTrue(sent_token in d_sent_token)
sent_tokens = util.sent_tokenize(o_text[1])
for sent_token in sent_tokens:
self.assertFalse(sent_token in d_sent_token)
def test_shouldSentenceTokenWork(self):
o_text = ['I am fine. how about you? the weather is hot']
right_tokens = ['I am fine.', 'how about you?', 'the weather is hot']
for text in o_text:
sent_tokens = util.sent_tokenize(text)
for (token, right_token) in zip(sent_tokens, right_tokens):
self.assertEqual(token, right_token)
def test_shouldContainMethodWithWordWork(self):
o_text = ['One needs to hydrate regularly.', 'intersteller was nice movie']
d_text = 'I am fine, how about you? this weather is really hot and humid. ' \
'One needs to hydrate regularly to survive.'
d_sent_token = util.sent_tokenize(d_text)
d_word_token = util.word_tokenize(d_text)
sent_tokens = util.sent_tokenize(o_text[0])
for sent_token in sent_tokens:
self.assertFalse(sent_token in d_sent_token)
for word in util.word_tokenize(sent_token):
self.assertTrue(word in d_word_token)
sent_tokens = util.sent_tokenize(o_text[1])
for sent_token in sent_tokens:
self.assertFalse(sent_token in d_sent_token)
for word in util.word_tokenize(sent_token):
self.assertFalse(word in d_word_token)
def calcDiff_Enhanced(user_id, should_clean=False):
"""
It calculates the longevity of the contribution of user in the next 50 revision
Args:
user_id (str): user id of the user.
Result:
the list of revisions contributed by user and the for each revision it has the Longevity value in no of revision and time.
"""
try:
with open('user_data_all/rev_list_' + user_id + '.json',
'r') as infile:
updated_data = json.loads(infile.read())
for row in updated_data:
print("Picking For Analysis Artcile,Parent,Revision: ",
[row['pageid'], row['parentid'], row['revid']])
capture_longevity = True
current_rev = util.read_file('rev_user/' + str(row['revid']))
if should_clean:
current_rev = util.cleanhtml(current_rev).strip()
if row['parentid'] == 0:
original_text = current_rev
else:
parent_rev = util.read_file('rev_user/' + str(row['parentid']))
if should_clean:
parent_rev = util.cleanhtml(parent_rev).strip()
original_text = util.findDiffRevised(parent_rev, current_rev)
original_text = list(v[1] for v in original_text)
original_text = [w for w in original_text if len(w) > 1]
original_text_clean = []
for contributions in original_text:
sent_toks_list = util.sent_tokenize(contributions)
for sent_tok in sent_toks_list:
original_text_clean.append(
util.stop_word_removal(sent_tok))
original_text = original_text_clean
total = 0
for txt in original_text:
total += len(txt)
row['contribLength'] = total
row['originaltext'] = original_text
next_revs = [i for i in row['next_rev']]
if total > 0:
print(original_text)
print(
"Performing Diff For Artcile,Parent,Revision: ",
[row['pageid'], row['parentid'], row['revid'], total])
index = 0
hasZero = False
lastUserID = 0
total_no_rev = 0
# finding total number of user turns
for rev in next_revs:
if rev['userid'] != lastUserID and rev[
'userid'] != row['userid']:
total_no_rev += 1
lastUserID = rev['userid']
print("Total Turns/ Contribution: ",
[total_no_rev, len(next_revs)])
lastUserID = 0
# finding achieved number of user turns
for rev in next_revs:
try:
next_rev = util.read_file('rev_user/' +
str(rev['revid']))
if should_clean:
next_rev = util.cleanhtml(next_rev)
d_text = util.getInsertedContentSinceParentRevision(
parent_rev, next_rev).strip()
ratio = util.textPreservedRatioBigramEnhanced(
original_text, d_text)
if rev['userid'] != lastUserID and rev[
'userid'] != row['userid']:
index += 1
lastUserID = rev['userid']
print("ratio: ", ratio)
if ratio == 0 and not hasZero:
hasZero = True
row['longevityRev'] = round(
index / total_no_rev, 2) * 100
row['matchRatio'] = ratio
row['totalContrib'] = total
print("in zero mode")
elif ratio >= 0.9 and hasZero:
hasZero = False
print("out zero mode")
if ratio < 0.90 and capture_longevity and not hasZero:
row['longevityRev'] = round(
index / total_no_rev, 2) * 100
row['matchRatio'] = ratio
row['totalContrib'] = total
capture_longevity = False
print("longevity-S: ", index)
break
except Exception as e:
print("file error", e)
if capture_longevity and not hasZero:
row['longevityRev'] = round(index / total_no_rev,
2) * 100
row['matchRatio'] = ratio
row['totalContrib'] = total
print("longevity-L: ", index)
if len(updated_data) > 0:
with open('user_data_all_b/rev_list_' + user_id + '-dp.json',
'w') as outfile:
json.dump(updated_data, outfile)
except Exception as e:
print("skipping diff as no contribution: ", e)