def tokenize(self, text):
text = _replace_html_entities(text)
for regexp, substitution in self.STARTING_QUOTES:
text = regexp.sub(substitution, text)
for regexp, substitution in self.PUNCTUATION:
text = regexp.sub(substitution, text)
text = " " + text + " "
# split contractions
for regexp, substitution in self.ENDING_QUOTES:
text = regexp.sub(substitution, text)
for regexp in self.CONTRACTIONS:
text = regexp.sub(r' \1 \2 ', text)
# handle emojis
for emoticon in list(EMOTICON_RE.finditer(text))[::-1]:
pos = emoticon.span()[0]
if text[pos - 1] != ' ':
text = text[:pos] + ' ' + text[pos:]
return text.split()
def tokenize(sentence, word_tokenizer = ReviewTokenizer(), stemmer = None, lower = False, remove_punc = False, remove_stopwords = False, remove_emoji = False, convert_neg = False):
tokens = word_tokenizer.tokenize(sentence)
# convert tokens to lowercase
if lower:
tokens = [token.lower() for token in tokens]
# remove stopword tokens
if remove_stopwords:
tokens = [token for token in tokens if token not in STOPWORDS]
# remove emoji tokens
if remove_emoji:
tokens = [token for token in tokens if not EMOTICON_RE.search(token)]
# NEG_SENTENCE_BOUNDARY_LIST
if convert_neg == True:
tokens = _convert_neg(tokens)
# remove punctuation tokens
if remove_punc:
tokens = [token for token in tokens if token not in PUNCTUATION]
# stem tokens
if stemmer:
tokens = [stemmer.stem(token) for token in tokens]
tokens = [SPECIAL_TOKEN_DICT[token] if SPECIAL_TOKEN_DICT.get(token, '') else token for token in tokens]
return tokens
def strip_emoticon(tweets):
no_emoticon = [[
re.sub(r'(?!\n)\s+', ' ', EMOTICON_RE.sub('', token))
for token in tweet if tweet
] for tweet in tweets]
return [[token for token in tweet if token] for tweet in no_emoticon]
def preprocess(self, text: str) -> str:
get_emoticon = self._emoticon_mapping.get
text = URLS_RE.sub("<url>", text)
text = USERNAMES_RE.sub("<user>", text)
text = HASHTAGS_RE.sub("<hashtag>", text)
text = NUMBERS_RE.sub("<number>", text)
text = EMOTICON_RE.sub(lambda m: get_emoticon(m.group()) or m.group(), text)
return text
def exec_simple(self, text):
tweet = TweetTokenizer()
# simple preprocessing
new_text = text.lower()
# tokenizing
tokens = tweet.tokenize(new_text)
# removing emoticons
new_tokens = []
for c in tokens:
if EMOTICON_RE.fullmatch(c) is not None: # deleting emoticons
continue
if c.startswith('#') or c.startswith(
'@'): # deleting User names and Hashtags
continue
if re.sub(r"(http[s]?|ftp):\S+", "", c) == '': # removing urls
continue
new_tokens.append(c)
# negating from 'not' to next punctuation sign
for i in range(0, len(new_tokens)):
if new_tokens[i] == 'not' or new_tokens[i] == 'dont' or new_tokens[
i].endswith('n\'t'):
for j in range(i + 1, len(new_tokens)):
if new_tokens[j] not in string.punctuation:
new_tokens[j] = new_tokens[j] + '_neg'
else:
break
# removing punctuation signs
punctuation = '!"$%&\'()*+,-./:;<=>?[\\]^_`{|}~]+'
new_text = []
found = False
for c in new_tokens:
for char in c:
if char in punctuation:
found = True
break
else:
break
if found:
found = False
continue
new_text.append(c)
# print(tokens)
for mod in self.queue:
tokens = mod(new_text).process()
str = ''
for s in tokens:
str += ' ' + s
return str
def _lowerize(word, keep_all_upper=False):
if EMOTICON_RE.search(word):
return word
elif word.isupper() and keep_all_upper:
return word
elif word == 'URL':
return word
elif word == '@USER':
return word
else:
return word.lower()
def segment_sent(text, emoji_tokenizer=TweetTokenizer()):
text = sanitise(text)
sentences = []
for sentence in sent_tokenize(text):
if EMOTICON_RE.search(sentence):
new_sentences = []
tokens = emoji_tokenizer.tokenize(sentence)
new_sentence = []
for token in tokens:
new_sentence.append(token)
if EMOTICON_RE.search(token) or token in '.?!':
new_sentences.append(' '.join(new_sentence))
new_sentence = []
if new_sentence:
new_sentences.append(' '.join(new_sentence))
sentences += new_sentences
else:
sentences.append(sentence)
if len(sentences) != 0:
if sentences[-1] in ['.', '!', '?']:
sentences[-2] = sentences[-2] + sentences[-1]
sentences = sentences[:-1]
return sentences
def classifyEmoticons(text):
# find all emoticons
emoticons = EMOTICON_RE.findall(text)
pos = any([emo in POS_EMOTICONS
for emo in emoticons]) or bool(POS_EMOJIS_RE.search(text))
neg = any([emo in NEG_EMOTICONS
for emo in emoticons]) or bool(NEG_EMOJIS_RE.search(text))
if pos and neg:
return 'N/A'
elif pos and not neg:
return 'pos'
elif neg and not pos:
return 'neg'
elif not pos and not neg:
return None
def emoticonList(corpus):
emolist = []
emocount = []
emoTorF = []
for a in corpus:
ct = [] # store count
TorF = [] # possive or negative
all_emoticons = EMOTICON_RE.findall(a)
ct.append(len(all_emoticons))
if (len(all_emoticons) != 0):
ed = EmoticonDetector()
all_emoticons.sort()
tf = (-1, 1)[ed.is_positive(all_emoticons[0])]
TorF.append(tf)
else:
TorF.append(0)
emolist.append(all_emoticons)
emocount.append(ct)
emoTorF.append(TorF)
return emocount, emoTorF, emolist
def tokenize(self, text):
"""
:param text: str
:rtype: list(str)
:return: a tokenized list of strings; concatenating this list returns\
the original string if `preserve_case=False`
"""
# Fix HTML character entities:
text = _replace_html_entities(text)
# Remove username handles
if self.strip_handles:
text = remove_handles(text)
# Normalize word lengthening
if self.reduce_len:
text = reduce_lengthening(text)
# Shorten problematic sequences of characters
safe_text = HANG_RE.sub(r"\1\1\1", text)
# Tokenize:
r"|<(?:[^\d>]+|:[A-Za-z0-9]+:)\w+>"
custom_Re = regex.compile(
r"""(%s)"""
% "|".join(
(
r":[^:\s]+:",
r"<:[^:\s]+:[0-9]+>",
r"<a:[^:\s]+:[0-9]+>",
r"<(?:[^\d>]+|:[A-Za-z0-9]+:)\w+>",
)
+ REGEXPS
),
regex.VERBOSE | regex.I | regex.UNICODE,
)
words = custom_Re.findall(safe_text)
# Possibly alter the case, but avoid changing emoticons like :D into :d:
if not self.preserve_case:
words = list(
map((lambda x: x if EMOTICON_RE.search(x) else x.lower()), words)
)
return words
def json2csv_preprocess(json_file,
outfile,
fields,
encoding='utf8',
errors='replace',
gzip_compress=False,
skip_retweets=True,
skip_tongue_tweets=True,
skip_ambiguous_tweets=True,
strip_off_emoticons=True,
remove_duplicates=True,
limit=None):
"""
Convert json file to csv file, preprocessing each row to obtain a suitable
dataset for tweets Semantic Analysis.
:param json_file: the original json file containing tweets.
:param outfile: the output csv filename.
:param fields: a list of fields that will be extracted from the json file and
kept in the output csv file.
:param encoding: the encoding of the files.
:param errors: the error handling strategy for the output writer.
:param gzip_compress: if True, create a compressed GZIP file.
:param skip_retweets: if True, remove retweets.
:param skip_tongue_tweets: if True, remove tweets containing ":P" and ":-P"
emoticons.
:param skip_ambiguous_tweets: if True, remove tweets containing both happy
and sad emoticons.
:param strip_off_emoticons: if True, strip off emoticons from all tweets.
:param remove_duplicates: if True, remove tweets appearing more than once.
:param limit: an integer to set the number of tweets to convert. After the
limit is reached the conversion will stop. It can be useful to create
subsets of the original tweets json data.
"""
with codecs.open(json_file, encoding=encoding) as fp:
(writer, outf) = outf_writer_compat(outfile, encoding, errors,
gzip_compress)
# write the list of fields as header
writer.writerow(fields)
if remove_duplicates == True:
tweets_cache = []
i = 0
for line in fp:
tweet = json.loads(line)
row = extract_fields(tweet, fields)
try:
text = row[fields.index('text')]
# Remove retweets
if skip_retweets == True:
if re.search(r'\bRT\b', text):
continue
# Remove tweets containing ":P" and ":-P" emoticons
if skip_tongue_tweets == True:
if re.search(r'\:\-?P\b', text):
continue
# Remove tweets containing both happy and sad emoticons
if skip_ambiguous_tweets == True:
all_emoticons = EMOTICON_RE.findall(text)
if all_emoticons:
if (set(all_emoticons) & HAPPY) and (set(all_emoticons)
& SAD):
continue
# Strip off emoticons from all tweets
if strip_off_emoticons == True:
row[fields.index('text')] = re.sub(
r'(?!\n)\s+', ' ', EMOTICON_RE.sub('', text))
# Remove duplicate tweets
if remove_duplicates == True:
if row[fields.index('text')] in tweets_cache:
continue
else:
tweets_cache.append(row[fields.index('text')])
except ValueError:
pass
writer.writerow(row)
i += 1
if limit and i >= limit:
break
outf.close()
def json2csv_preprocess(json_file, outfile, fields, encoding='utf8', errors='replace',
gzip_compress=False, skip_retweets=True, skip_tongue_tweets=True,
skip_ambiguous_tweets=True, strip_off_emoticons=True, remove_duplicates=True,
limit=None):
"""
Convert json file to csv file, preprocessing each row to obtain a suitable
dataset for tweets Semantic Analysis.
:param json_file: the original json file containing tweets.
:param outfile: the output csv filename.
:param fields: a list of fields that will be extracted from the json file and
kept in the output csv file.
:param encoding: the encoding of the files.
:param errors: the error handling strategy for the output writer.
:param gzip_compress: if True, create a compressed GZIP file.
:param skip_retweets: if True, remove retweets.
:param skip_tongue_tweets: if True, remove tweets containing ":P" and ":-P"
emoticons.
:param skip_ambiguous_tweets: if True, remove tweets containing both happy
and sad emoticons.
:param strip_off_emoticons: if True, strip off emoticons from all tweets.
:param remove_duplicates: if True, remove tweets appearing more than once.
:param limit: an integer to set the number of tweets to convert. After the
limit is reached the conversion will stop. It can be useful to create
subsets of the original tweets json data.
"""
with codecs.open(json_file, encoding=encoding) as fp:
(writer, outf) = outf_writer_compat(outfile, encoding, errors, gzip_compress)
# write the list of fields as header
writer.writerow(fields)
if remove_duplicates == True:
tweets_cache = []
i = 0
for line in fp:
tweet = json.loads(line)
row = extract_fields(tweet, fields)
try:
text = row[fields.index('text')]
# Remove retweets
if skip_retweets == True:
if re.search(r'\bRT\b', text):
continue
# Remove tweets containing ":P" and ":-P" emoticons
if skip_tongue_tweets == True:
if re.search(r'\:\-?P\b', text):
continue
# Remove tweets containing both happy and sad emoticons
if skip_ambiguous_tweets == True:
all_emoticons = EMOTICON_RE.findall(text)
if all_emoticons:
if (set(all_emoticons) & HAPPY) and (set(all_emoticons) & SAD):
continue
# Strip off emoticons from all tweets
if strip_off_emoticons == True:
row[fields.index('text')] = re.sub(r'(?!\n)\s+', ' ', EMOTICON_RE.sub('', text))
# Remove duplicate tweets
if remove_duplicates == True:
if row[fields.index('text')] in tweets_cache:
continue
else:
tweets_cache.append(row[fields.index('text')])
except ValueError:
pass
writer.writerow(row)
i += 1
if limit and i >= limit:
break
outf.close()
pd.set_option('display.max_colwidth', -1)
pd.set_option('display.max_columns', 10)
DEFAULT_DATA_FILE = "./data/sample_data.json"
IMAGES_DIRECTORY = './images'
DEFAULT_SEED = 42
STOPWORDS = set(
stopwords.words('english') +
["'ve','s", "one", "use", "would", "get", "also"]) - {
'not', 'no', 'won', 'more', 'above', 'very', 'against', 'again'
}
PUNCTUATION = string.punctuation + '...'
SPECIAL_TOKEN_DICT = {"n't": 'not'}
flatten = lambda l: [item for sublist in l for item in sublist]
is_word = lambda token: not (EMOTICON_RE.search(token) or token in string.
punctuation or token in STOPWORDS)
NEG_SENT_BOUND_RE = re.compile(EMOTICON_RE.pattern + '|' +
'|'.join(['\.', '\:', '\;', '\!', '\?', '\,']))
NEG_WORD_RE = re.compile(r"(?:^(?:never|no|nothing|nowhere|noone|none|not)$)")
def main(data_file, seed):
# set seed
np.random.seed(seed)
# load in a pd.df
data = [json.loads(line) for file in data_file for line in file]
df = pd.DataFrame.from_dict(data)
pd.set_option('display.max_colwidth', -1)
pd.set_option('display.max_columns', 10)
DEFAULT_DATA_FILE = "./data/sample_data.json"
IMAGES_DIRECTORY = './images'
DEFAULT_SEED = 42
STOPWORDS = set(stopwords.words('english') + ["'ve", "'d", "'s", "one", "use", "would", "get", "also"]) - {'not', 'no', 'won', 'more', 'above', 'very', 'against', 'again'}
PUNCTUATION = string.punctuation + '...'
SPECIAL_TOKEN_DICT = {"n't": 'not'}
boundary_punc = '.:;!?,'
NEG_SENT_BOUND_RE = re.compile(EMOTICON_RE.pattern + '|' + '|'.join([re.escape(punc) for punc in boundary_punc]))
NEG_WORD_RE = re.compile(r"(?:^(?:never|no|nothing|nowhere|noone|none|not)$)")
flatten = lambda l: [item for sublist in l for item in sublist]
is_word = lambda token: not(EMOTICON_RE.search(token) or token in string.punctuation or token in STOPWORDS)
class ReviewTokenizer(TreebankWordTokenizer):
_contractions = MacIntyreContractions()
CONTRACTIONS = list(map(re.compile, _contractions.CONTRACTIONS2 + _contractions.CONTRACTIONS3))
PUNCTUATION = [
(re.compile(r'([,])([^\d])'), r' \1 \2'),
(re.compile(r'([,])$'), r' \1 '),
(re.compile(r'\.\.\.'), r' ... '),
(re.compile(r'[;@#&]'), r' \g<0> '),
# Handles the final period
(re.compile(r'([^\.])(\.)([\]\)}>"\']*)\s*$'), r'\1 \2\3 '),
(re.compile(r'[?!]'), r' \g<0> '),
(re.compile(r"([^'])' "), r"\1 ' "),
i = 0
url_re = re.compile(URLS, re.VERBOSE | re.I | re.UNICODE)
hashtag_re = re.compile('(?:^|\s)[＃#]{1}(\w+)', re.UNICODE)
#mention_re = re.compile('(?:^|\s)[＠@]{1}([^\s#<>[\]|{}]+)', re.UNICODE) # To include more complete names
mention_re = re.compile('(?:^|\s)[＠@]{1}(\w+)', re.UNICODE)
with open(text_file, 'r') as text_reader, open(
words_file, 'w', encoding='utf-8') as words_writer, open(
emo_file, 'w', encoding='utf-8') as emo_writer, open(
hash_file, 'w', encoding='utf-8') as hash_writer, open(
at_file, 'w', encoding='utf-8') as at_writer, open(
link_file, 'w', encoding='utf-8') as link_writer:
for line in text_reader:
line = line.rstrip().lower()
hashs = hashtag_re.findall(line)
ats = mention_re.findall(line)
links = url_re.findall(line)
line = clean(line, hashs, ats, links)
emoticons = emo_re.findall(line)
emojis = [w for w in line if w in emoji.UNICODE_EMOJI]
words = re.findall('[a-záéíóúñ][a-záéíóúñ_-]+',
line) #Revisar para remover ats, hashs y links
words_writer.write(' '.join(w for w in words) + '\n')
emo_writer.write(' '.join(w for w in emoticons + emojis) + '\n')
hash_writer.write(' '.join(w for w in hashs) + '\n')
at_writer.write(' '.join(w for w in ats) + '\n')
link_writer.write(' '.join(w for w in links) + '\n')
i += 1
print(i)
def strip_emoticon(tweets):
no_emoticon = [[re.sub(r'(?!\n)\s+', ' ', EMOTICON_RE.sub('', token)) for token in tweet if tweet] for tweet in tweets]
return [[token for token in tweet if token] for tweet in no_emoticon]
