def validated_tweet(self, data):
''' A bunch of checks to determine whether the tweet is valid.
Also returns emojis contained by the tweet.
'''
# Ordering of validations is important for speed
# If it passes all checks, then the tweet is validated for usage
# Skips incomplete tweets
if len(data) <= 9:
return False, []
text = data[9]
if self.ignore_retweets and RETWEETS_RE.search(text):
return False, []
if self.ignore_url_tweets and URLS_RE.search(text):
return False, []
if self.ignore_mention_tweets and MENTION_RE.search(text):
return False, []
if self.wanted_emojis is not None:
uniq_emojis = np.unique(extract_emojis(text, self.wanted_emojis))
if len(uniq_emojis) == 0:
return False, []
else:
uniq_emojis = []
if self.non_english_user_set is not None and \
non_english_user(data[1], self.non_english_user_set):
return False, []
return True, uniq_emojis
def generate_validation_data():
file_number = 0
data_val = []
label_val = []
while 1:
print("Loading Validation file: " + validation_files[file_number])
with open(os.path.join(path_to_train,
validation_files[file_number])) as json_file:
data_preprocess = json_file.readlines()
data_preprocess = [pre_process(x) for x in data_preprocess]
data_preprocess = filter(lambda y: y != '', data_preprocess)
random.shuffle(data_preprocess)
for i in range(len(data_preprocess)):
text = data_preprocess[i]
emojis = extract_emojis(text, wanted_emojis)
emojis = np.unique(emojis)
for l in range(len(emojis)):
text = text.replace(emojis[l], '')
for j in range(len(emojis)):
data_val.append(text)
label_val.append(wanted_emojis_index.get_loc(emojis[j]))
if (i % 1024 == 1023 or i == len(data_preprocess) - 1):
sequences = tokenizer.texts_to_sequences(data_val)
X_val = pad_sequences(sequences,
maxlen=max_sequence_length)
Y_val = np.eye(64)[label_val]
print("Loaded Validation " + str(len(X_val)) +
" records from" + validation_files[file_number])
yield (X_val, Y_val)
data_val = []
label_val = []
file_number = (file_number + 1) % len(validation_files)
def prepare_data_from_excel(path_to_load):
final_texts = []
final_labels = []
data = pd.read_excel(path_to_load)
content = data["Message"].tolist()
content = [convert_to_unicode(x) for x in content]
content = filter(lambda y: y != '', content)
for i in range((len(content))):
text = content[i]
emojis = extract_emojis(text, wanted_emojis)
emojis = np.unique(emojis)
for l in range(len(emojis)):
text = text.replace(emojis[l], '')
for j in range(len(emojis)):
final_texts.append(text)
final_labels.append(wanted_emojis_index.get_loc(emojis[j]))
data = prepare_texts(final_texts)
final_labels = np.eye(64)[final_labels]
return data, final_labels
def prepare_data_from_json(file_to_load):
final_texts = []
final_labels = []
print("Loading file: " + file_to_load)
with open(os.path.join(path_to_train, file_to_load)) as json_file:
content = json_file.readlines()
content = [pre_process(x) for x in content]
content = filter(lambda y: y != '', content)
for i in range((len(content))):
text = content[i]
emojis = extract_emojis(text, wanted_emojis)
emojis = np.unique(emojis)
for l in range(len(emojis)):
text = text.replace(emojis[l], '')
for j in range(len(emojis)):
final_texts.append(text)
final_labels.append(wanted_emojis_index.get_loc(emojis[j]))
data = prepare_texts(final_texts)
final_labels = np.eye(64)[final_labels]
return data, final_labels