def get_recall_unordered(recall_list, query_list):
"""
Determine the fraction of queries in the recall list that were captured by the query list without concern for
word order, capitalization, or punctuation. Differences in apostrophes (single quotes) will still be considered.
:param recall_list: list containing all queries from a recall data set
:param query_list: list containing all automatically generated queries
:return: fraction of recall queries captured
"""
tokenizer = RegexpTokenizer(r'[a-zA-Z\']+')
query_token = []
for q in query_list:
l = tokenizer.tokenize(q.lower())
sorted(l, key=str.lower)
query_token.append(l)
num = 0
for q in recall_list:
l = tokenizer.tokenize(q.lower())
sorted(l, key=str.lower)
if l in query_token:
num += 1
if len(recall_list) == 0:
return 0, 0
return float(num) / float(len(recall_list)), num
def remove_stop_word_punctuation(sentence):
'''
removes stop word and punctuation from given text
param sentence: user provided text
type sentence: str
returns: preprocessed sentence
rtype: str
'''
stop_words = set(stopwords.words('english'))
word_tokens = []
for token in word_tokenize(sentence):
for t in wordpunct_tokenize(token):
# remove punctuations
tokenizer = RegexpTokenizer(r'\w+')
r = tokenizer.tokenize(t)
if r:
word_tokens.append(tokenizer.tokenize(t)[0])
# remove stop words
filtered_sentence = [w for w in word_tokens if not w.lower() in stop_words]
return ' '.join(x for x in filtered_sentence)
def getText(self):
filename = self.textEdit.toPlainText()
symb_remove = RegexpTokenizer(r'\w+')
list1 = symb_remove.tokenize(filename)
print("-----------------------------------")
filename = self.textEdit_2.toPlainText()
symb_remove = RegexpTokenizer(r'\w+')
list2 = symb_remove.tokenize(filename)
print("following is the wordlist of the file")
print(list1)
print(list2)
sims = []
initialList = []
for word1, word2 in product(list1, list2):
syns1 = wordnet.synsets(word1)
print(syns1)
syns2 = wordnet.synsets(word2)
print(syns2)
for word1 in syns1:
for word2 in syns2:
s = word1.wup_similarity(word2)
if str(s) == 'None':
s = 0
initialList.append(s)
print(str(word1) + " second word" + str(word2))
print(s)
print(initialList)
def tokenize(self, attr):
accepted = {
'title': self.title,
'description': self.description,
'cve': self.cve,
'cwe': self.cwe,
'refs': self.refs,
'dsk': self.dsk
}
matcher = {
'title': r'\w+[-\w+]*',
'description': r'\w+[-\w+]*',
'cve': r'CVE[\s|-]\d+[\s|-]\d+'
}
if attr not in accepted.keys():
return 'It is not possible to tokenize this plugin attribute.'
tokenizer = RegexpTokenizer(matcher[attr])
stop = stopwords.words('english')
final = []
if attr == 'title' or attr == 'description':
intermediate = tokenizer.tokenize(accepted[attr])
final = [i.lower() for i in intermediate if i not in stop]
elif attr == 'cve':
intermediate = tokenizer.tokenize(','.join(accepted[attr]))
final = [i.lower().replace(' ', '-') for i in intermediate if i not in stop]
return final
def get_tokens(dict_element):
# Remove stop words from data and perform initial
# cleanup for feature extraction
query = dict_element['query']
desc = dict_element['product_description']
title = dict_element['product_title']
stop = stopwords.words('english')
pattern = r'''(?x) # set flag to allow verbose regexps
([A-Z]\.)+ # abbreviations, e.g. U.S.A.
| \$?\d+(\.\d+)?%? # numbers, incl. currency and percentages
| \w+([-']\w+)* # words w/ optional internal hyphens/apostrophe
| @((\w)+([-']\w+))*
| [+/\-@&*] # special characters with meanings
'''
#pattern = r'[+/\-@&*#](\w+)|(\w+)'
tokenizer = RegexpTokenizer(pattern)
#tokenizer = RegexpTokenizer(r'\w+')
query_tokens = tokenizer.tokenize(query)
query_tokens = map(lambda x:x.lower(),query_tokens)
desc_tokens = tokenizer.tokenize(desc)
desc_tokens = [x.lower() for x in desc_tokens if x.lower() not in stop]
title_tokens = tokenizer.tokenize(title)
title_tokens = [x.lower() for x in title_tokens if x.lower() not in stop]
return query_tokens, title_tokens, desc_tokens
def summarize(text):
tokenizer = RegexpTokenizer(r'\w+')
formatted_text = tokenizer.tokenize(text)
sentence_list = nltk.sent_tokenize(text)
stopwords = nltk.corpus.stopwords.words('english')
word_frequencies = {}
for word in formatted_text:
if word not in stopwords:
if word not in word_frequencies:
word_frequencies[word] = 1
else:
word_frequencies[word] += 1
max_freq = max(word_frequencies.values())
for word in word_frequencies.keys():
word_frequencies[word] = word_frequencies[word]/max_freq
sentence_scores = {}
for sent in sentence_list:
for word in tokenizer.tokenize(sent.lower()):
if word in word_frequencies:
if len(sent.split(' ')) < 30:
if sent not in sentence_scores:
sentence_scores[sent] = word_frequencies[word]
else:
sentence_scores[sent] += word_frequencies[word]
import heapq
summary_sentences = heapq.nlargest(7, sentence_scores, key=sentence_scores.get)
summary = ' '.join(summary_sentences)
return summary
def Preprocessing(df, contractions):
pd.options.mode.chained_assignment = None
contractionsDict = {}
for i in contractions['data']:
contractionsDict[i[0]] = i[1]
# remove url
df['sentence'] = df['sentence'].str.replace('http\S+|www.\S+',
'',
case=False)
# remove number
df['sentence'] = df['sentence'].str.replace('\d+', '')
# remove hashtags
df['sentence'] = df['sentence'].str.replace('#(\w+)', '')
# change all text with contraction
for index, row in df.iterrows():
row[1] = ' '.join([
str(x) for x in [
contractionsDict[t] if t in contractionsDict.keys() else t
for t in [e.lower() for e in row[1].split()]
]
])
# remove stopword
stop_words = []
for word in stopwords.words('english'):
stop_words.append(word) if ('not' not in word
and 'no' not in word) else stop_words
# remove punctuation
tokenizer = RegexpTokenizer(r'\w+')
for index, row in df.iterrows():
word_tokens = tokenizer.tokenize(row[1])
row[1] = ' '.join(
[w for w in word_tokens if not w.lower() in stop_words])
# using lemmetizer
wordnet_lemmatizer = WordNetLemmatizer()
for index, row in df.iterrows():
row[1] = ' '.join(
wordnet_lemmatizer.lemmatize(t) for t in row[1].split())
# remove non-english word
english_words = set(nltk.corpus.words.words())
for index, row in df.iterrows():
word_tokens = tokenizer.tokenize(row[1])
row[1] = " ".join(w for w in word_tokens
if w.lower() in english_words or not w.isalpha())
# remove non-alphabetic characters
for index, row in df.iterrows():
word_tokens = tokenizer.tokenize(row[1])
row[1] = " ".join(w for w in word_tokens if w.isalpha())
return df
def clean_text(text, stop_words):
'''Make text lowercase, remove mentions, remove links, convert emoticons/emojis to words, remove punctuation
(except apostrophes), tokenize words (including contractions), convert contractions to full words,
remove stop words.'''
# make text lowercase
text = text.lower()
# remove mentions
text = re.sub("(@[A-Za-z0-9]+)", "", text)
# remove links
text = re.sub(r'http\S+', '', text)
text = re.sub(r'pic\.\S+', '', text)
# convert emoticons
emoticons = load_dict_emoticons()
words = text.split()
words_edit = [
emoticons[word] if word in emoticons else word for word in words
]
tweet = ' '.join(words_edit)
# convert emojis
text = emoji.demojize(text)
text = text.replace(':', ' ') # separate emojis-words with space
# remove punctuation
text = text.replace('...', ' ') # special cases
text = text.replace('-', ' ')
text = text.translate(
str.maketrans('', '', '!"$%&*()+,./;<=>?@[\\]^_`{|}~'))
# tokenize words
tokenizer = RegexpTokenizer("(#?[a-zA-Z]+[0-9]*(?:'[a-zx]+)?)")
words = tokenizer.tokenize(text)
# convert contractions
contractions = load_dict_contractions()
words = text.split()
words_edit = [
contractions[word] if word in contractions else word for word in words
]
text = ' '.join(words_edit)
# remove stop words and lemmatize
lemmatizer = WordNetLemmatizer()
words = tokenizer.tokenize(text)
words = [
lemmatizer.lemmatize(word) for word in words if word not in stop_words
]
text = ' '.join(words)
return text
def remove_punctuation(statement):
"""
Remove any punctuation from the statement text
"""
# Make tokenizer
tokenizer = RegexpTokenizer(r"\w+")
#make a list of words without punctuation
temp_text = tokenizer.tokenize(statement.text)
statement.text = " ".join(tokenizer.tokenize(statement.text))
print(statement.text)
return statement
def preprocess(problem: str, code: str):
"""Normalize and tokenize raw problem text and extract normalized comment lines from code file.
Tokenized using RegexpTokenizer from NLTK.
Args:
problem (str): raw problem text from db
code (str): raw code file, pre-encoded
Returns:
tuple: list of tokenized lines of problem text, list of tokenized lines of comments
"""
tokenizer = RegexpTokenizer(r'\w+')
# normalize problem text
# convert to printable representation to reveal carriage return characters
problem = repr(problem)
# remove html tags
problem = re.sub(r'<.*?>', '', problem)
# remove carriage return characters
problem = re.sub(r'\\r', ' ', problem)
# revert from printable representation
problem = eval(problem)
problem = problem.split('.')
problem = [tokenizer.tokenize(line.lower()) for line in problem]
# trim newline and tab characters
code = re.sub(r'[\r|\n|\t]+', '\n', code)
# trim multiple spaces (and to handle codes with leading space as indentation)
# this, however, turns all multiple whitespace into one, so we need to strip the remaining leading whitespace afterwards
code = re.sub(r' +', ' ', code)
# remove remaining leading whitespace
code = '\n'.join([line.strip() for line in code.split('\n')])
# extract comments from code using regex pattern
# regex pattern from https://www.regexpal.com/94246
re_pattern = r'/\*[\s\S]*?\*/|([^:]|^)//.*$'
matcher = re.compile(re_pattern, re.MULTILINE)
matches = matcher.finditer(code)
# iterate through found matches while removing newline and leading-trailing whitespaces
comments = []
for match in matches:
comment_line = match.group()
comments.append(tokenizer.tokenize(comment_line.lower()))
# get only the codes by removing comments
code_only = matcher.sub("", code).split('\n')
return problem, comments, code_only
def add_to_index(self, document, doc_id):
# parser = HTMLParser(text=document['data'])
text = document['data']
# print(1)
nlp = Russian()
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(text)
tokens = [token.lower() for token in tokens]
tmp_text = ' '.join(tokens)
if len(tokens) > 10e5:
return
self.doc_iter += 1
nlp.max_length = 10e7
doc_text = nlp(tmp_text, disable=['ner', 'parser'])
lemmas = []
# for lemma in tokens:
for s in doc_text:
lemma = s.lemma_
lemmas.append(lemma)
# if lemma not in set(stopwords.words('russian')) \
# and lemma not in set(stopwords.words('english')) \
# and len(lemma) > 1:
# lemmas.append(lemma)
freq = FreqDist(lemmas)
for k, v in freq.most_common():
if k not in self.global_index:
self.global_index[k] = []
self.global_index[k].append((doc_id, v))
def gen_counts(path_corpus, list_corpus):
""" creates np array, for each corpus file how many words
in that document """
# create output
counts_corpus = np.zeros(len(list_corpus))
fp = None
txt = u''
tokens = []
tokenizer = RegexpTokenizer(ur'\w+')
count = 0
every = 500
for f in list_corpus:
# read in text
fp = codecs.open(path_corpus+f, 'r', "utf-8", errors="ignore")
txt = fp.read()
txt = txt.lower()
fp.close()
# tokenize
tokens = tokenizer.tokenize(txt)
counts_corpus[list_corpus.index(f)] = len(tokens)
# count interations
if count % every == 0:
print(count)
count += 1
return counts_corpus
def read_all_txt_orig(directory):
all_s = []
for file in os.listdir(directory):
full_path = os.path.join(directory, file)
if not file.endswith(".txt"):
continue
with open(full_path) as f:
captions = f.read().split('\n')
for cap in captions:
if len(cap) == 0 or len(cap) == 1:
continue
cap = cap.replace("\ufffd\ufffd", " ")
# picks out sequences of alphanumeric characters as tokens
# and drops everything else
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(cap.lower())
# print('tokens', tokens)
if len(tokens) == 0:
print('cap', cap)
continue
tokens_new = []
for t in tokens:
if t == 'thisbirdhasadarkgreybelly':
print(123)
t = t.encode('ascii', 'ignore').decode('ascii')
if len(t) > 0:
tokens_new.append(t)
all_s.append(" ".join(tokens_new) + "\n")
return all_s
def filter_sentence(sentence):
tokenizer = RegexpTokenizer(r'\w+')
word_tokens = tokenizer.tokenize(sentence)
filtered_words = [w for w in word_tokens if not w in stop_words]
snowball_result_set = [snowball_stemmer.stem(word) for word in filtered_words]
return snowball_result_set
def tensor_vec_pipline(data, word_index, max_len):
#Create data maxtrix to be fed to the keras model
print("Creating data to feed to tensorflow")
df_len = len(data)
indexing_matrix = np.zeros((df_len, max_len), dtype = 'int32')
r_inc = 0
tokenizer = RegexpTokenizer(r'\w+')
for index, row in data.iterrows():
sentence = row['sentence']
sen_tokenize = tokenizer.tokenize(sentence)
c_inc = 0
for word in sen_tokenize:
try:
indexing_matrix[r_inc][c_inc] = word_index[word]
except Exception as e:
#print(e, word)
if (str(e) == word):
indexing_matrix[r_inc][c_inc] = 0
continue
c_inc = c_inc + 1
r_inc = r_inc + 1
print("Run complete")
return indexing_matrix
def french_tokenizer(text):
from nltk import RegexpTokenizer
tokenizer = RegexpTokenizer(r"(?u)\b\w\w+\b")
toks = tokenizer.tokenize(text)
# We also lemmatize!
# toks = [fr_lexicon.get(t, t) for t in toks]
return toks
def read_and_clean_training_data(file):
"""
The following function reads the training data and split them based the label
:param file:
:return:
"""
with open(file, encoding="utf8", errors="ignore") as f:
stop_words = set(stopwords.words('english'))
lines = f.readlines()
x_train = []
y_train = []
tokenizer = RegexpTokenizer(r'\w+')
for i in range(len(lines)):
line = lines[i]
if i == 0:
continue
else:
data, label = line.split('\t')
label = label.strip()
tokenized_data = tokenizer.tokenize(data)
cleaned_data = [
word.lower() for word in tokenized_data
if not word.isdigit() and word != "ml"
and word not in stop_words
]
final_data = " ".join(cleaned_data)
x_train.append(final_data)
y_train.append(label)
return x_train, y_train
def clean_test_data(test_data, test_labels):
"""
Using that function you are able to read the test files and make the predictions
:param test_data: test data file
:param test_labels: test labels file
:return:
"""
xtest = []
ytest = []
with open(test_data, encoding="utf8", errors="ignore") as data:
stop_words = set(stopwords.words('english'))
tokenizer = RegexpTokenizer(r'\w+')
lines = data.readlines()
for i in range(len(lines)):
line = lines[i]
if i == 0:
continue
else:
tokenized_line = tokenizer.tokenize(line)
cleaned_data = [
word.lower() for word in tokenized_line
if not word.isdigit() and word != "ml"
and word not in stop_words
]
xtest.append(" ".join(cleaned_data))
with open(test_labels, encoding="utf8", errors="ignore") as labels:
ydata = labels.readlines()
for label in ydata:
ytest.append(label.strip())
return xtest, ytest
class RawLemmaTokenizer(object):
def __init__(self):
self.tokenizer = RegexpTokenizer(u'(?u)\\b\\w\\w+\\b')
self.wnl = WordNetLemmatizer()
def __call__(self, doc):
return [self.wnl.lemmatize(t) for t in self.tokenizer.tokenize(doc)]
def regex_tokenizer(self, sent, whole_sent=False):
regex_tokenizer = RT('\w+|\[M:.*?\]|[\(\)\.\,;\?\!]|\S+')
tokens = regex_tokenizer.tokenize(sent)
i = 0
j = len(tokens) - 1
# combine abbreviations with their period;
# separation is a result of tokenizing the sentence
while i < j:
if re.match(self.abbrev_pattern,
tokens[i]) and tokens[i + 1] == '.':
tokens[i:i + 2] = [''.join(tokens[i:i + 2])]
j -= 1
i += 1
# return tokenized sentence minus stopword and short words
if not whole_sent:
return [
t for t in tokens if not t in self.stop_words and len(t) > 2
]
# return entire tokenized sentence
else:
return [t for t in tokens]
def gen_vocab(vocab_fname, path):
print("\ngen_vocab:{}".format(vocab_fname))
""" reads in a csv file,
outputs as python list in given path
as pickled object. unicode.
Also add unigrams for every line"""
# open file pointer
f = codecs.open(path+vocab_fname, 'r', "utf-8")
# output list
concepts = []
# read in lines
for line in f.readlines():
concepts = concepts + line.lower().strip("\n").split(',')
# from observation the concept lists all had ''
while ('' in concepts):
concepts.remove('')
# add unigrams to concepts. does not preserve order of list
unigrams = set()
set_concepts = set(concepts)
tokenizer = RegexpTokenizer(ur'\w+')
for phrase in concepts:
unigrams.update(tokenizer.tokenize(phrase))
set_concepts.update(unigrams)
return list(set_concepts)
def analyze_dataset():
l_sentences = []
with open('/Users/miljan/PycharmProjects/thesis-shared/data/pang_and_lee_data/rt-negative.txt') as file1:
r = reader(file1, dialect='excel-tab')
for row in r:
l_sentences.append(row[0])
with open('/Users/miljan/PycharmProjects/thesis-shared/data/pang_and_lee_data/rt-positive.txt') as file2:
r = reader(file2, dialect='excel-tab')
for row in r:
l_sentences.append(row[0])
# chunk the given text into sentences
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
d_lengths = defaultdict(int)
tokenizer2 = RegexpTokenizer(r'\w+')
# clean sentences from punctuation
l_sentences = [''.join(ch for ch in sent if ch not in set(string.punctuation)) for sent in l_sentences]
l_sentences = [len(tokenizer2.tokenize(sen)) for sen in l_sentences]
total_sent = len(l_sentences)
d_lengths = Counter(l_sentences)
print total_sent
lengths = sorted(d_lengths.iteritems(), key=lambda key_value: int(key_value[0]))
plot(lengths)
def create_bag_of_words(document_list):
"""
Creates a bag of words representation of the document list given. It removes
the punctuation and the stop words.
:type document_list: list[str]
:param document_list:
:rtype: list[list[str]]
:return:
"""
tokenizer = RegexpTokenizer(r'\w+')
cached_stop_words = set(stopwords.words("english"))
body = []
processed = []
# remove common words and tokenize
# texts = [[word for word in document.lower().split() if word not in stopwords.words('english')]
# for document in reviews]
for i in range(0, len(document_list)):
body.append(document_list[i].lower())
for entry in body:
row = tokenizer.tokenize(entry)
processed.append([word for word in row if word not in cached_stop_words])
return processed
def clean_text(t):
sentence = t.lower()
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(sentence)
filtered_words = filter(
lambda token: token not in stopwords.words('english'), tokens)
return " ".join(filtered_words)
def processText(self,Estr):
# ① 去除HTML标签
content = re.sub(r'<[^>]*>', ' ', Estr)
# ② 除去标点符号,等非字母的字符
tokenizer = RegexpTokenizer(r'[a-z]+')
raw = str(content).lower()
content = tokenizer.tokenize(raw)
# ③ 去除停用词
# 获取英语的停用词表
en_stop = stopwords.words('english') # get_stop_words('en')
# 获取自己的停用词表
# file = os.getcwd()+"\\..\\datasets\\stopwords.txt"
# f = open(file, "r")
# mystopwords = f.read()
# mystopwords= mystopwords.split('\n')
# for word in mystopwords:
# en_stop.add(word)
# 去除文本中的停用词
stopped_tokens = [i for i in content if not i in en_stop]
# ④ 按长度过滤
content = [i for i in stopped_tokens if len(i) > 2]
return content
def prep_text_to_stem(text):
"""
Remove partes indesejadas como números e palavras na stop_list. Além disso adicionar # ao final da
palavra a fim de facilitar no stems de uma única letra
:param text:
:return:
"""
text = list(filter(lambda x: type(x) == str, text))
tokenizer = RegexpTokenizer(r'\w+', flags=re.UNICODE)
tokens = tokenizer.tokenize(' '.join(text).lower())
new_tokens = []
stop_list = Counter(tokens).most_common(300)
stop_list = [tup[0] for tup in stop_list]
stop_list.append('series([],')
for token in tokens:
if token not in stop_list:
token = ''.join(
[letter for letter in token if not letter.isdigit()])
for pun in punct:
token.replace(pun, '')
new_token = token + '#'
new_tokens.append(new_token)
return ' '.join(new_tokens)
def text2sents(text, lemmatize=False, stemmer=None):
"""
converts a text into a list of sentences consisted of normalized words
:param text: list of string to process
:param lemmatize: if true, words will be lemmatized, otherwise -- stemmed
:param stemmer: stemmer to be used, if None, PortedStemmer is used. Only applyed if lemmatize==False
:return: list of lists of words
"""
sents = sent_tokenize(text)
tokenizer = RegexpTokenizer(r'\w+')
if lemmatize:
normalizer = WordNetLemmatizer()
tagger = PerceptronTagger()
elif stemmer is None:
normalizer = PorterStemmer()
else:
normalizer = stemmer
sents_normalized = []
for sent in sents:
sent_tokenized = tokenizer.tokenize(sent)
if lemmatize:
sent_tagged = tagger.tag(sent_tokenized)
sent_normalized = [normalizer.lemmatize(w[0], get_wordnet_pos(w[1])) for w in sent_tagged]
else:
sent_normalized = [normalizer.stem(w) for w in sent_tokenized]
sents_normalized.append(sent_normalized)
return sents_normalized
class Preprocessor(object):
def __init__(self, max_workers=4):
self.max_workers = max_workers
self.tokenizer = RegexpTokenizer(r'\w+')
self.en_stopwords = set(get_stop_words('en'))
self.p_stemmer = PorterStemmer()
def preprocess_doc(self, doc):
tokens = self.tokenizer.tokenize(doc.lower())
stopped_tokens = [i for i in tokens if i not in self.en_stopwords]
stemmed_tokens = [self.p_stemmer.stem(i) for i in stopped_tokens]
return stemmed_tokens
def process_docs(self, doc_list):
with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
return [self.preprocess_doc(doc) for doc in doc_list]
def preprocess_doc_with_url(self, doc_with_url):
with ProcessPoolExecutor(max_workers=self.max_workers) as executor:
url, content = doc_with_url
return url, self.preprocess_doc(content)
def process_docs_with_urls(self, urldoc_list):
return [self.preprocess_doc_with_url(urldoc) for urldoc in urldoc_list]
def _get_ngram_features(infile, ngram_size):
"""
Returns a dictionary containing ngrams and counts observed in a given file
:param infile: file to be analysed
:param ngram_size: ngram size
:return: dict of ngrams/counts
"""
# tokenizer which remove punctuation
tokenizer = RegexpTokenizer(r'\w+')
# dictionary on ngrams and counts
d_ngrams = defaultdict(int)
# stopwords
stops = set(stopwords.words("english"))
# lemmatizer for stemming
lemmatizer = WordNetLemmatizer()
# load train data
with open(infile) as tsv:
file_reader = reader(tsv, dialect="excel-tab")
# skip title line
file_reader.next()
for line in file_reader:
s_text = line[2]
# remove punctuation and tokenize
l_text = tokenizer.tokenize(s_text)
# remove stopwords and stem
l_text = [lemmatizer.lemmatize(word) for word in l_text if word not in stops]
# get the ngrams for the given line
l_temp = ngrams(l_text, ngram_size)
for ngram in l_temp:
d_ngrams[ngram] += 1
return d_ngrams
def prepare_text(text: pd.Series) -> pd.Series:
"""
Naive approach to text cleaning. Strip out HTML, then do relatively strict
preparation (lemmatization, stopwords)
:param text: series of all relevant text data
"""
# first, remove html tags
wo_html = text.apply(lambda x: BeautifulSoup(x, "lxml").text)
tokenizer = RegexpTokenizer(r'\w+')
stopword_set = set(stopwords.words('english'))
lmtzr = WordNetLemmatizer()
clean_text = []
pbar = tqdm(range(len(text)), desc='clean_text')
for d in wo_html:
dlist = d.lower()
dlist = tokenizer.tokenize(dlist)
dlist = list(set(dlist).difference(stopword_set))
# filter tokens
filtered_tokens = []
for token in dlist:
if re.search('^[a-zA-Z]+$', token) and len(token) >= 4:
filtered_tokens.append(token)
# lemmatize
stems = [lmtzr.lemmatize(t) for t in filtered_tokens]
final_stems = [stem for stem in stems if len(stem) > 3]
clean_text.append(final_stems)
pbar.update()
pbar.close()
return clean_text
def preprocessing(self):
self.df = pd.read_csv('static/models/resampled_comments_1.csv')
self.comments = self.df[['comment', 'rating', 'sentiment']]
self.comments['comment'] = self.comments['comment'].map(
lambda x: x.lower())
toknizer = RegexpTokenizer(r'''\w'|\w+|[^\w\s]''')
token = self.comments.apply(
lambda row: toknizer.tokenize(row['comment']), axis=1)
stop_words = set(stopwords.words('french'))
stop_token = token.apply(
lambda x: [item for item in x if item not in stop_words])
stemmer = SnowballStemmer(language='french')
stemm = stop_token.apply(lambda x: [stemmer.stem(y) for y in x])
lemmatizer = FrenchLefffLemmatizer()
lemm = stemm.apply(lambda x: [lemmatizer.lemmatize(y) for y in x])
for i in range(len(lemm)):
lemm[i] = ' '.join(lemm[i])
self.comments['lemmatiser_com'] = lemm
data = self.comments[['comment', 'lemmatiser_com', 'sentiment']]
self.df = pd.DataFrame(data)
return self.df
def test():
global N, words, network
print 'In testing.'
gettysburg = """Four score and seven years ago our fathers brought forth on this continent, a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure. We are met on a great battle-field of that war. We have come to dedicate a portion of that field, as a final resting place for those who here gave their lives that that nation might live. It is altogether fitting and proper that we should do this. But, in a larger sense, we can not dedicate -- we can not consecrate -- we can not hallow -- this ground. The brave men, living and dead, who struggled here, have consecrated it, far above our poor power to add or detract. The world will little note, nor long remember what we say here, but it can never forget what they did here. It is for us the living, rather, to be dedicated here to the unfinished work which they who fought here have thus far so nobly advanced. It is rather for us to be here dedicated to the great task remaining before us -- that from these honored dead we take increased devotion to that cause for which they gave the last full measure of devotion -- that we here highly resolve that these dead shall not have died in vain -- that this nation, under God, shall have a new birth of freedom -- and that government of the people, by the people, for the people, shall not perish from the earth."""
tokenizer = RegexpTokenizer('\w+')
gettysburg_tokens = tokenizer.tokenize(gettysburg)
samples = []
for token in gettysburg_tokens:
word = token.lower()
if word not in ENGLISH_STOP_WORDS and word not in punctuation:
samples.append(word)
dist = FreqDist(samples)
V = Vol(1, 1, N, 0.0)
for i, word in enumerate(words):
V.w[i] = dist.freq(word)
pred = network.forward(V).w
topics = []
while len(topics) != 5:
max_act = max(pred)
topic_idx = pred.index(max_act)
topic = words[topic_idx]
if topic in gettysburg_tokens:
topics.append(topic)
del pred[topic_idx]
print 'Topics of the Gettysburg Address:'
print topics
def french_tokenizer(text):
from nltk import RegexpTokenizer
tokenizer = RegexpTokenizer(r"(?u)\b\w\w+\b")
toks = tokenizer.tokenize(text)
# We also lemmatize!
# toks = [fr_lexicon.get(t, t) for t in toks]
return toks
def analyze_dataset():
l_sentences = []
with open(
'/Users/miljan/PycharmProjects/thesis-shared/data/pang_and_lee_data/rt-negative.txt'
) as file1:
r = reader(file1, dialect='excel-tab')
for row in r:
l_sentences.append(row[0])
with open(
'/Users/miljan/PycharmProjects/thesis-shared/data/pang_and_lee_data/rt-positive.txt'
) as file2:
r = reader(file2, dialect='excel-tab')
for row in r:
l_sentences.append(row[0])
# chunk the given text into sentences
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
d_lengths = defaultdict(int)
tokenizer2 = RegexpTokenizer(r'\w+')
# clean sentences from punctuation
l_sentences = [
''.join(ch for ch in sent if ch not in set(string.punctuation))
for sent in l_sentences
]
l_sentences = [len(tokenizer2.tokenize(sen)) for sen in l_sentences]
total_sent = len(l_sentences)
d_lengths = Counter(l_sentences)
print total_sent
lengths = sorted(d_lengths.iteritems(),
key=lambda key_value: int(key_value[0]))
plot(lengths)
def clean_text(text, stop_words):
'''Make text lowercase, tokenize words and words with apostrophes, convert contractions to full words,
lemmatize by POS tag, remove stop words and words shorter than 3 letters.'''
# make text lowercase
text = text.lower().replace("’", "'")
# initial tokenization to remove non-words
tokenizer = RegexpTokenizer("([a-z]+(?:'[a-z]+)?)")
words = tokenizer.tokenize(text)
# convert contractions
contractions = load_dict_contractions()
words = [contractions[word] if word in contractions else word for word in words]
text = ' '.join(words)
# remove stop words, lemmatize using POS tags, and remove two-letter words
lemmatizer = WordNetLemmatizer()
words = [lemmatizer.lemmatize(word, get_wordnet_pos(word)) for word in nltk.word_tokenize(text) \
if word not in stop_words]
# removing any words that got lemmatized into a stop word
words = [word for word in words if word not in stop_words]
words = [word for word in words if len(word) > 2]
text = ' '.join(words)
return text
def analyze_articles():
json_document = _read_json_articles()
l_articles = [
json_document[i]['_source']['content']
for i in range(len(json_document))
]
# chunk the given text into sentences
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
d_lengths = defaultdict(int)
tokenizer2 = RegexpTokenizer(r'\w+')
total_sent = 0
for article in l_articles:
l_sentences = tokenizer.tokenize(article)
# clean sentences from punctuation
l_sentences = [
''.join(ch for ch in sent if ch not in set(string.punctuation))
for sent in l_sentences
]
l_sentences = [len(tokenizer2.tokenize(sen)) for sen in l_sentences]
total_sent += len(l_sentences)
d_counts = Counter(l_sentences)
for key in d_counts.keys():
d_lengths[str(key)] += d_counts[key]
print total_sent
lengths = sorted(d_lengths.iteritems(),
key=lambda key_value: int(key_value[0]))
plot(lengths)
def frequencyAnalyse(polarised_tweets : Dict):
positive_words = {}
negative_words = {}
tokenizer = RegexpTokenizer(r'\w+')
stop_words = list(stopwords.words('english'))
for i in polarised_tweets:
word_pit = tokenizer.tokenize(polarised_tweets[i][0])
tags = nltk.pos_tag(word_pit)
for word in tags:
if word[0] in positive_words:
positive_words[word[0]] += 1
continue
elif word[0] in negative_words:
negative_words[word[0]] += 1
continue
if len(word[0]) < 3:
continue
if word[0].lower() in stop_words:
continue
if word[1] in ['JJ']:
if polarised_tweets[i][1] > 0.2: #Positive
positive_words[word[0].lower()] = 1
elif polarised_tweets[i][1] < -0.2: #Negative
negative_words[word[0].lower()] = 1
for w in sorted(negative_words, key=negative_words.get, reverse=True):
print(w, negative_words[w])
return (positive_words, negative_words)
def get_documents_text(act_id, **kwargs):
"""
Returns the concatenated, tag-stripped text of all documents related to act_id
"""
db_conn = kwargs['db']
italian_stops = set(stopwords.words('italian'))
cursor = db_conn.cursor(MySQLdb.cursors.DictCursor)
sql = """
select d.testo
from opp_documento as d
where d.atto_id=%s
"""
cursor.execute(sql, act_id)
rows = cursor.fetchall()
cursor.close()
testo = u''
for row in rows:
# strip html tags from texts, if present
testo += unicode(
strip_tags(
row['testo']
)
)
# remove stopwords
tokenizer = RegexpTokenizer("[\w]+")
words = tokenizer.tokenize(testo)
filtered_testo = " ".join([word for word in words if word.lower() not in italian_stops])
return filtered_testo
def tokenize(self, string):
# Supression des espaces non nécessaires
space = re.compile(r' +')
string = re.sub(space, ' ', string)
# Harmonisation des numéros de téléphone
tel = re.compile(
r'(?P<sep1>0[0-9])( |/+|\-|\\+)(?P<sep2>[0-9]{2})( |/+|\.|\-|\\+)(?P<sep3>[0-9]{2})( |/+|\.|\-|\\+)(?P<sep4>[0-9]{2})( |/+|\.|\-|\\+)(?P<sep5>[0-9]{2})'
)
string = tel.sub(r'\g<sep1>.\g<sep2>.\g<sep3>.\g<sep4>.\g<sep5>',
string)
# Tokenisation
# Le tokenizer supprime automatiquement les caractères suivant isolés : `^ ° ¤ ¨
# Reconnait comme token :
# - Email
# - Site web, nom de domaine, utilisateur etc
# - Numéro de téléphone réduit
# - Nom composé
# - Mot courant
# - Ponctuation
tokenizer = RegexpTokenizer(
r'''([Aa]ujourd'hui|\w+'|[a-zA-ZÀ-Ÿà-ÿ0-9_\.\-]+@[a-zA-ZÀ-Ÿà-ÿ0-9\-\.]+\.[a-zA-ZÀ-Ÿà-ÿ0-9]+|[a-zA-ZÀ-Ÿà-ÿ0-9:@%/;$~_?\+\-=\\\.&\|£€]+[a-zA-ZÀ-Ÿà-ÿ0-9#@%/$~_?\+\-=\\&\|£€]+|[\wÀ-Ÿà-ÿ]+[/\-][\wÀ-Ÿà-ÿ]+|[\wÀ-Ÿà-ÿ0-9]+|\.\.\.|[\(\)\[\]\{\}\"\'\.,;\:\?!\-\_\*\#\§=+<>/\\])'''
)
tokens = tokenizer.tokenize(string)
return tokens
def tokenize(text):
"""
Input: "Body of text...:
Output: [word, ...] list of tokenized words matching regex '\w+'
"""
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(text)
return tokens
def tokenize(self, text):
"""
tokenise text using nltk RegexpTokenizer
:param text:
:return: list of tokens
"""
tokenizer = RegexpTokenizer(self.pattern)
tokens = tokenizer.tokenize(text)
return tokens
class StemTokenizer(object):
def __init__(self):
self.wnl = PorterStemmer()
self.mytokenizer = RegexpTokenizer('\\b\\w+\\b')
def __call__(self, doc):
#return [self.wnl.stem(t) for t in word_tokenize(doc)]
return [self.wnl.stem(t) for t in self.mytokenizer.tokenize(doc)]
class StemTokenizer(object):
def __init__(self):
from nltk import RegexpTokenizer
from nltk.stem import PorterStemmer
self.wnl = PorterStemmer()
self.mytokenizer = RegexpTokenizer('\\b\\w+\\b')
def __call__(self, doc):
return [self.wnl.stem(t) for t in self.mytokenizer.tokenize(doc)]
def tokenize(self, text):
"""
:param tweet_list:
:type list:
:return: tokens
This tokenizer uses the nltk RegexpTokenizer.
"""
tokenizer = RegexpTokenizer(self.pattern)
tokens = tokenizer.tokenize(text)
return tokens
def __call__(self, doc ):
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
#tokenizer = RegexpTokenizer(r'\w+')
tokenizer = RegexpTokenizer(r'[a-zA-Z]+')
#words=[self.wnl.lemmatize(t) for t in word_tokenize(doc)]
words=[self.wnl.lemmatize(t) for t in tokenizer.tokenize(doc)]
mystops=(u'youtube',u'mine',u'this',u'that','facebook','com','google','www','http','https')
stop_words=set(stopwords.words('english'))
stop_words.update(mystops)
stop_words=list(stop_words)
return [i.lower() for i in words if i not in stop_words]
def tokenizeWords(corpus_root):
wordlists = PlaintextCorpusReader(corpus_root, '.*')
tokenizer = RegexpTokenizer(r'\w+')
# for fileid in wordlists.fileids():
# sentimentText=wordlists.raw(fileid).lower()
# tokenizedWords=tokenizer.tokenize(sentimentText)
# tokenizedTextWithoutStopWords=removeAllStopWords(tokenizedWords)
#
# print(tokenizedTextWithoutStopWords)
# if "positive" in corpus_root:
# print("positive documents")
# #posfeats.update(word_feats(tokenizedTextWithoutStopWords),'pos')
# #posfeats =posfeats+[word_feats(tokenizedTextWithoutStopWords), 'pos']
# posfeats[word_feats(tokenizedTextWithoutStopWords)]='pos'
#
# if "negative" in corpus_root:
# negfeats.update(word_feats(tokenizedTextWithoutStopWords),'neg')
if "negative" in corpus_root:
negfeats = [(word_feats(removeAllStopWords(tokenizer.tokenize(wordlists.raw(f).lower()))), 'neg') for f in wordlists.fileids()]
if "positive" in corpus_root:
posfeats = [(word_feats(removeAllStopWords(tokenizer.tokenize(wordlists.raw(f).lower()))), 'pos') for f in wordlists.fileids()]
print(posfeats)
def tokenize_and_stem(doc):
tokenizer = RegexpTokenizer(r'\w+')
# create English stop words list
en_stop = get_stop_words('en')
# Create p_stemmer of class PorterStemmer
p_stemmer = PorterStemmer()
tokens = tokenizer.tokenize(doc)
clean = [token.lower() for token in tokens if token.lower() not in en_stop and len(token) > 2]
final = [p_stemmer.stem(word) for word in clean]
return final
def gen_doc_term_counts(path_corpus, list_corpus, list_vocab):
print("\ngen_doc_term_counts:{}".format(path_corpus))
""" generates document-term matrix given a path to a
corpus and common vocab
"""
num_docs = len(list_corpus)
num_terms = len(list_vocab)
doc_term = np.zeros((num_docs, num_terms))
counts_corpus = np.zeros(num_docs)
# generate (dict) compiled regex's
re_c_vocab = gen_regex_c(list_vocab)
tokenizer = RegexpTokenizer(ur'\w+')
# iterate over files
fp = None
txt = u''
r = None
num = 0.0
tokens = []
count = 0
every = 50
start= timeit.default_timer()
checkpoint = 0.0
for i in range(num_docs):
fp = codecs.open(path_corpus+list_corpus[i], 'r', "utf-8", errors="ignore")
txt = fp.read()
txt = txt.lower()
fp.close()
# tokenize
tokens = tokenizer.tokenize(txt)
counts_corpus[i] = len(tokens)
# count number terms
for j in range(num_terms):
r = re_c_vocab[ list_vocab[j] ]
num = len(r.findall(txt, re.UNICODE))
doc_term[i,j] = num
if (count % every == 0):
checkpoint = timeit.default_timer()
print(count, round(checkpoint-start, 2))
count += 1
return (doc_term, counts_corpus)
def __call__(self, doc ,string_tokenize='[a-zA-Z0-9]+'):
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from nltk.corpus import wordnet as wn
#tokenizer = RegexpTokenizer(r'\w+')
tokenizer = RegexpTokenizer(string_tokenize)
#words=[self.wnl.lemmatize(t) for t in word_tokenize(doc)]
words=[self.wnl.lemmatize(t) for t in tokenizer.tokenize(doc)]
mystops=(u'youtube',u'mine',u'this',u'that')
stop_words=set(stopwords.words('english'))
stop_words.update(mystops)
stop_words=list(stop_words)
words1= [i.lower() for i in words if i not in stop_words]
words2= list(set(list({l.name() for word in words1 for s in wn.synsets(word) for l in s.lemmas()})+words1))
return [i.lower() for i in words2 if i not in stop_words]
def build_vector(text, neutral):
# We tokenize the text
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(text)
if neutral:
tokens = pos_tag(tokens) # we add POS tag
forbidden_pos = ['RB', 'RBS', 'RBR', 'CC', 'CD', 'DT', 'EX', 'IN', 'LS', 'PDT', 'PRP', 'PRP$', 'RP', 'SYM', 'TO', 'WDT', 'WP', 'WP$', ]
# We build the document vector
vector = set()
for couple in tokens:
if neutral:
if (couple[1] in forbidden_pos):
continue
vector.add(lemmatize(couple[0]))
else:
vector.add(lemmatize(couple))
return vector
def test():
gt = GetTweets()
documents = gt.get_hashtag('ferguson', count=20)
documents += gt.get_hashtag('police', count=21)
print 'Query:', documents[-1]
tokenizer = RegexpTokenizer('\w+')
vols = []
for doc in documents:
samples = []
for token in tokenizer.tokenize(doc):
word = token.lower()
if word not in ENGLISH_STOP_WORDS and word not in punctuation:
samples.append(word)
vols.append(volumize(FreqDist(samples)))
vectors = [ doc_code(v) for v in vols[:-1] ]
query_vec = doc_code(vols[-1])
sims = [ cos(v, query_vec) for v in vectors ]
m = max(sims)
print m, documents[sims.index(m)]
def analyze_articles():
json_document = _read_json_articles()
l_articles = [json_document[i]['_source']['content'] for i in range(len(json_document))]
# chunk the given text into sentences
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
d_lengths = defaultdict(int)
tokenizer2 = RegexpTokenizer(r'\w+')
total_sent = 0
for article in l_articles:
l_sentences = tokenizer.tokenize(article)
# clean sentences from punctuation
l_sentences = [''.join(ch for ch in sent if ch not in set(string.punctuation)) for sent in l_sentences]
l_sentences = [len(tokenizer2.tokenize(sen)) for sen in l_sentences]
total_sent += len(l_sentences)
d_counts = Counter(l_sentences)
for key in d_counts.keys():
d_lengths[str(key)] += d_counts[key]
print total_sent
lengths = sorted(d_lengths.iteritems(), key=lambda key_value: int(key_value[0]))
plot(lengths)
def create_bag_of_words(document_list):
"""
Creates a bag of words representation of the document list given. It removes
the punctuation and the stop words.
:type document_list: list[str]
:param document_list:
:rtype: list[list[str]]
:return:
"""
tokenizer = RegexpTokenizer(r'\w+')
tagger = nltk.PerceptronTagger()
cached_stop_words = set(stopwords.words("english"))
cached_stop_words |= {
't', 'didn', 'doesn', 'haven', 'don', 'aren', 'isn', 've', 'll',
'couldn', 'm', 'hasn', 'hadn', 'won', 'shouldn', 's', 'wasn',
'wouldn'}
body = []
processed = []
for i in range(0, len(document_list)):
body.append(document_list[i].lower())
for entry in body:
row = tokenizer.tokenize(entry)
tagged_words = tagger.tag(row)
nouns = []
for tagged_word in tagged_words:
if tagged_word[1].startswith('NN'):
nouns.append(tagged_word[0])
nouns = [word for word in nouns if word not in cached_stop_words]
processed.append(nouns)
return processed
class StemmingTokenizer(object):
def __init__(self):
self.stemmer = PorterStemmer()
self.tokenizer = RegexpTokenizer(u'(?u)\\b[a-z]+\\-*[a-z]+|\\b(?u)\\b[a-z]\\b')
def __call__(self, doc):
return [self.stemmer.stem(tokens.lower()) for tokens in self.tokenizer.tokenize(doc)]
nyt_labels.append(line[4])
nyt.close()
f = open('/home/mikhail/Documents/research/hierarchical_classification/Inter_Observ/Interv.csv', 'a')
writer = csv.writer(f)
countNCT = 0
countSCRT = 0
ope = 0
try:
for i in range(0,len(nyt_data),1):
text = nyt_data[i]+" "+nyt_data1[i]
try:
observ = RegexpTokenizer("NCT[0-9]{8}")
obs = observ.tokenize(nyt_labels[i])
if len(obs) > 0:
page = urllib2.urlopen('http://clinicaltrials.gov/show/'+obs[0]+'?resultsxml=true')
document = ElementTree.parse(page)
page_content = page.read()
study_design = document.findtext('study_type')
writer.writerow((text.replace("\'","").replace("\"","").replace("\\","").replace("\/",""),study_design))
print("NORMAL"+" "+nyt_labels[i]+" "+study_design)
countNCT +=1
except Exception,e:
print(nyt_labels[i])
#error2_writer.writerow((encode_id,encode_date,encoded_user,encoded_str,NCT))
print e
pass
pformat(lcwd)
def filter_stopwords(words):
important_words = filter(lambda x: x not in stopwords.words('english'), words)
return important_words
#Parsing the command line arguments for the filename
parser = argparse.ArgumentParser(description = 'Process a text file.')
parser.add_argument('filename', type=str, help='pathname to that file')
parser.add_argument('cut_off', type=int, help='cut off value for the list output')
args = parser.parse_args()
print args
filename = args.filename
#open the file
pp = open(filename)
CUTOFF = args.cut_off
# string of the text
text = pp.read()
# the list of words
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(text)
print "Without filtering out the stop words"
process_words(tokens)
print "With stop words filtering"
fil_token = filter_stopwords(tokens)
process_words(fil_token)
tokens.append(ngram)
return tokens
indonesian = []
malaysian = []
tamil = []
others = []
lang_dict = {'indonesian' : indonesian, 'malaysian' : malaysian, 'tamil' : tamil, 'others' : others}
file_content = getFileContents('input.train.txt')
for line in file_content:
try:
#tokens = nltk.word_tokenize(line)
tokenizer = RegexpTokenizer(r'\w+')
tokens = tokenizer.tokenize(line)
language = tokens[0]
del tokens[0]
ngrams = getNgrams(tokens)
for gram in ngrams:
lang_dict[language].append(gram)
except UnicodeEncodeError:
pass
for key,value in lang_dict.items():
with open('stuff.txt', 'a') as f:
f.write('\n')
f.write(key)
f.write('\n')
for k in value:
try:
def stemming(doc):
wnl = PorterStemmer()
mytokenizer = RegexpTokenizer('\\b\\w+\\b')
return [wnl.stem(t) for t in mytokenizer.tokenize(doc)]
from nltk.corpus import cess_esp as cess
from nltk import RegexpTokenizer
import nltk
import pickle
# My sentences
sentence = "hola, hola, soy Pedro ¿como te llamas?."
tokenizer = RegexpTokenizer(r'\w+')
tokenized_words = tokenizer.tokenize(sentence)
# Dec train/test
train = None
test = None
cess_sents = cess.tagged_sents()
try:
with open('test_pickles/test_data.pickle', 'rb') as fa:
div = pickle.load(fa)
train = cess_sents[:div]
test = cess_sents[div+1:]
except FileNotFoundError as a:
# training data
print("dumping train/test")
div = len(cess_sents)*90//100
train = cess_sents[:div]
test = cess_sents[div+1:]
with open('test_pickles/test_data.pickle', 'wb') as fb:
pickle.dump(div, fb)
#####
#
