def parse_senstiment(self, text):
tokenizer = RegexTokenizer()
tokens = tokenizer.split('всё очень плохо') # хз
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
return model.predict([text], k=2)[0]
class SentimentAnalyzer:
def __init__(self):
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
def _map_sentiment(self, sentiment):
if sentiment == "positive":
return 1
if sentiment == "negative":
return -1
else:
return 0
def sentiment_label_dataframe(self, df: DataFrame) -> DataFrame:
df['sentiment'] = None
for i, row in df.iterrows():
results = self.model.predict([row['text']], k=30)
sentiment = max(list(results[0].items()),
key=lambda prob: prob[1])[0]
sentiment = self._map_sentiment(sentiment)
df.at[i, 'sentiment'] = sentiment
return df
def sentiment_label_sentence(self, sentence: str):
results = self.model.predict([sentence], k=30)
sentiment = max(list(results[0].items()), key=lambda prob: prob[1])[0]
return self._map_sentiment(sentiment)
def validation_comment(
self, film_id, user_id
): # валидация отзыва, если он проходит проверку то comm_check = 1
x = self.conn.execute(
'SELECT rating1, rating2, rating3, rating4, rating5, comm_text '
'FROM Comments WHERE film_id = (?) and user_id = (?)', (
film_id[0],
user_id,
)).fetchall()
sr = 0
for r in x[0][0:5]:
sr = sr + r
sr = sr / 5
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
result = model.predict([x[0][5]], k=5)
dl = result[0]['neutral'] + result[0]['negative'] + result[0][
'positive']
res = 10 * (result[0]['neutral'] / dl - result[0]['negative'] / dl +
result[0]['positive'] / dl)
if abs(sr - res) <= valid_const:
self.set_comment(film_id, user_id, 'comm_check', 1)
self.conn.commit()
self.set_comment(film_id, user_id, 'second_check', 0)
self.conn.commit()
def graph():
file = filedialog.askopenfilename(filetypes=(("Text files", "*.txt"),
("all files", "*.*")))
f = open(file)
raw = f.read()
sentences = nltk.sent_tokenize(raw)
command = 'download'
arguments = ['fasttext-social-network-model']
if command == 'download':
downloader = DataDownloader()
for filename in arguments:
if filename not in AVAILABLE_FILES:
raise ValueError(f'Unknown package: {filename}')
source, destination = AVAILABLE_FILES[filename]
destination_path: str = os.path.join(DATA_BASE_PATH, destination)
if os.path.exists(destination_path):
continue
downloader.download(source=source, destination=destination)
else:
raise ValueError('Unknown command')
import dostoevsky
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = sentences
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
positive_values_all = [
sentiment.get('positive')
for message, sentiment in zip(messages, results)
]
positive_values = [
0.0 if value == None else value for value in positive_values_all
]
negative_values_all = [
sentiment.get('negative')
for message, sentiment in zip(messages, results)
]
negative_values = [
0.0 if value == None else value for value in negative_values_all
]
summary = (len(negative_values))
n_value = np.array(negative_values)
p_value = np.array(positive_values)
counts_value = np.arange(summary)
plt.plot(counts_value, p_value, n_value)
plt.show()
def analyze(all_news):
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
texts = [news['title'] + ' ' + news['article'] for news in all_news]
results = model.predict(texts)
for i in range(len(all_news)):
all_news[i]['tone'] = results[i]
return all_news
def analysis_data(messages):
res = []
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
print(messages)
i = 0
for message in messages:
data = model.predict([message], k=2)[0]
i += 1
for key, value in data.items():
if round(value, 1) == 1:
res.append(key)
if len(res) != i:
res.append('neutral')
return res
def get_sentiment(text):
if not text:
return 5
model = FastTextSocialNetworkModel(tokenizer=RegexTokenizer(),
lemmatize=True)
results = model.predict([text], k=2)
result = 0
result = result + (min(results[0]['positive'], 1)
if 'positive' in results[0] else 0)
result = result - (min(results[0]['negative'], 1)
if 'negative' in results[0] else 0)
return round((result + 1) * 5)
def text_analysis(all_comments):
texts = [comment['comment'] for comment in all_comments]
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(texts, k=5)
for i, sentiment in enumerate(results):
all_comments[i]['positive'] = sentiment['positive']
all_comments[i]['negative'] = sentiment['negative']
all_comments[i]['neutral'] = sentiment['neutral']
return all_comments
class DostN:
def __init__(self):
self.db = DB()
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
self.tokens = self.tokenizer.split('всё очень плохо')
def ready_msg(self, messages, nnID):
results = self.model.predict(messages, k=2)
sentiments = []
for message, sentiment in zip(messages, results):
sentiments.append(sentiment)
try:
pos = sentiments[0]['positive']
self.db.setPos(nnID, pos)
except Exception as e:
print(e, 22)
pos = 0
self.db.setPos(nnID, pos)
try:
neg = sentiments[0]['negative']
self.db.setNegative(nnID, neg)
except Exception as e:
print(e, 29)
neg = 0
self.db.setNegative(nnID, neg)
try:
ne = sentiments[0]['neutral']
self.db.setNeutral(nnID, ne)
except Exception as e:
print(e, 36)
ne = 0
self.db.setNeutral(nnID, ne)
return (sentiments)
def test_tonality():
test = [
'''Ужасное расположение и распределение товаров. Два уровня и на каждом свои кассы. Чтобы купить разные группы товаров нужно отстоять две очереди.''',
'''Это не шаурма это ужас,куча майонеза,лук одна кожура верхний слой который мы при готовке выкладываем, картофель фри из пачек сухой, мясо порезано тонкими пластами, не пойму как оно приготавливалось явно не на гриле, мясо было не свежее, в итоге самый съедобный оказался лаваш, не рекомендую.''',
'''Рядом с домом, вкусная картошечка и обалденные молочные коктейли и довольно быстрое обслуживание, приятные кассиры''',
'''Замечательный телефон, пользуюсь им уже 2 года, очень нравится!''',
'''Был в этом магазине в прошлом году, больше туда приходить не собираюсь, некомпетентные продавцы, плохое обслуживание((''',
'''Идеальный мастер!:)''',
'''Уроды, как можно так поступить с человеком, просто ужас?!'''
]
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(test, k=2)
for i, el in enumerate(zip(test, results)):
token, sentiment = el
print_tonality(test[i], sentiment.items())
def sentiment_analysis(data):
langid.set_languages(['en', 'ru'])
lang = langid.classify(data['text'][0])[0]
if lang == 'ru':
labels = data['from'].unique()
msg_df = data.loc[data.text != '']
messages_1 = list(msg_df.text[msg_df['from'] == labels[0]])
messages_2 = list(msg_df.text[msg_df['from'] == labels[1]])
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results_1 = model.predict(messages_1, k=2)
sentiments_1 = []
for sentiment in results_1:
# привет -> {'speech': 1.0000100135803223, 'skip': 0.0020607432816177607}
# люблю тебя!! -> {'positive': 0.9886782765388489, 'skip': 0.005394937004894018}
# малолетние дебилы -> {'negative': 0.9525841474533081, 'neutral': 0.13661839067935944}]
tone = 0
if 'positive' in sentiment:
tone += sentiment['positive']
if 'negative' in sentiment:
tone -= sentiment['negative']
sentiments_1.append(tone)
results_2 = model.predict(messages_2, k=2)
sentiments_2 = []
for sentiment in results_2:
tone = 0
if 'positive' in sentiment:
tone += sentiment['positive']
if 'negative' in sentiment:
tone -= sentiment['negative']
sentiments_2.append(tone)
return sentiments_1, sentiments_2
def return_sentiment(shcode):
response = requests.get(
get_url(f'https://www.instagram.com/p/{shcode}/?__a=1'))
data = json.loads(response.content)
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
sentim = []
edges = data['graphql']['shortcode_media']['edge_media_preview_comment'][
'edges']
for com in edges:
# print(com['node']['text'])
results = model.predict([com['node']['text']],
k=len(com['node']['text']))
for x in results:
sentim.append(list(x.keys())[0])
return sentim
def processSentiment(message, stage):
duration = '0.0'
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
phrases = []
for phrase in message:
duration = phrase['end_time']
for item in phrase['alternatives']:
phrases.append(item['transcript'])
results = model.predict(phrases, k=2)
for phrase, sentiments in zip(phrases, results):
for sentiment in sentiments.keys():
if sentiment not in ['neutral', 'skip']:
answer = sentiment
result = stages[stage][answer]
return [answer, result, float(duration[0:-1:])]
def tonal_analize(string):
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = [string]
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
negative = 0
positive = 0
if 'positive' in sentiment:
positive += sentiment['positive']
if 'negative' in sentiment:
negative += sentiment['negative']
delta = positive - negative
return delta
def get_sent(x):
def one_hot_encode_sent(x):
"""
(pos, neg, neu)
"""
if x[0] == 'positive':
return (1, 0, 0)
elif x[0] == 'negative':
return (0, 1, 0)
else:
return (0, 0, 1)
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(x, k=1)
results = [(list(r.keys())[0], list(r.values())[0]) for r in results]
results = list(map(one_hot_encode_sent, results))
results = [
results[i] if x[i] != 'EMPTY_TEXT' else (0, 0, 0)
for i in range(len(results))
]
return [pd.Series(x) for x in zip(*results)] # return three series
def begin():
file = filedialog.askopenfilename(filetypes=(("Text files", "*.txt"),
("all files", "*.*")))
f = open(file)
raw = f.read()
sentences = nltk.sent_tokenize(raw)
command = 'download'
arguments = ['fasttext-social-network-model']
if command == 'download':
downloader = DataDownloader()
for filename in arguments:
if filename not in AVAILABLE_FILES:
raise ValueError(f'Unknown package: {filename}')
source, destination = AVAILABLE_FILES[filename]
destination_path: str = os.path.join(DATA_BASE_PATH, destination)
if os.path.exists(destination_path):
continue
downloader.download(source=source, destination=destination)
else:
raise ValueError('Unknown command')
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = sentences
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
analysis_line = '\n', message, '\n', '->', '\n', sentiment, '\n'
text.insert(END, analysis_line)
def main():
db_con = init_sync()
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
with db_con:
cur = db_con.cursor()
# Запрос к предложениям выдленным томитой
cur.execute("SELECT id, text FROM filtered WHERE text IS NOT NULL")
filtreds = [{x[0]: x[1]} for x in cur.fetchall() if len(x[1]) > 0]
tonality = []
for filtred in filtreds:
results = model.predict(filtred.values(), k=2)
for i, el in enumerate(zip(filtred.values(), results)):
token, sentiment = el
tonality.clear()
for key, value in filtred.items():
id_filtered = key
text = value
for item in sentiment.items():
tonality.append(item)
insert_tonality(db_con, id_filtered, tonality)
print_tonality(text, tonality)
def _prepare_model():
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
return model
import re
import nltk
import json
import logging
from navec import Navec
from slovnet import Syntax
from razdel import sentenize, tokenize
from dostoevsky.tokenization import UDBaselineTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
meaningful_tags = ['positive', 'negative']
# init models:
sentenizer = FastTextSocialNetworkModel(tokenizer=UDBaselineTokenizer())
class SimpleExtractor:
def __init__(self, path_to_navec_data, path_to_syntax_data):
self.navec = Navec.load(path_to_navec_data)
self.syntax = Syntax.load(path_to_syntax_data).navec(self.navec)
def is_values_lower(self, source, target):
for v in source:
for d in list(target):
if v > d:
return False
return True
def extract_sentnces_with_names(self, text):
text = re.sub("\\s+", " ", text)
def __init__(self):
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
from .parse_kaldi import open_text
tokenizer = RegexTokenizer() # токенизатор текста
text_model = FastTextSocialNetworkModel(
tokenizer=tokenizer) # модель анализа тональности
to_numbers = {
'negative': 0,
'neutral': 1,
'positive': 2,
'skip': 1,
'speech': 1,
}
def choose_sentiment(pred):
return sorted(pred, key=lambda x: pred[x])[-1]
def predict_sentiment(texts):
preds = text_model.predict(texts)
return list(map(lambda pred: choose_sentiment(pred), preds))
def predict_by_text_file(text_path):
text = open_text(text_path).strip()
if text == '':
return 1
return predict_by_texts([text])[0]
from typing import Dict
from dostoevsky.models import FastTextSocialNetworkModel
from dostoevsky.tokenization import RegexTokenizer
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
class SentimentData:
high_negative: float
low_negative: float
neutral: float
skip: float
speech: float
positive: float
def __init__(self, sentiment_data: Dict[str, float] = {'negative': 0,
'neutral': 0,
'positive': 0,
'speech': 0,
'skip': 0}):
if 'negative' in sentiment_data:
if sentiment_data['negative'] > 0.59:
self.high_negative = sentiment_data['negative']
self.low_negative = 0
else:
self.high_negative = 0
self.low_negative = sentiment_data['negative']
if 'neutral' in sentiment_data:
class CommentsResearcher:
""" Класс для анализа тональности комментариев """
def __init__(self):
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
def get_sentiment(self, file):
""" Опредение тональности комментариев
Возвращает 2 значения:
- distribution (dict)
словарь в количеством комментариев в разных категориях
- sentiments (dict)
словарь с тональностью каждого комментария
Подробная информация о результатах записывается в файл с именем сообщества.
Файл сохраняется в папку 'reports'.
Аргументы:
file (_io.TextIOWrapper): текстовой файл с комментариями (разделитель \t)
"""
comments = file.read().split('\t')
f_name = os.path.basename(file.name)
# получение тональности для комментариев с помощью модели
results = self.model.predict(comments[0:-1], k=1)
output = []
distribution = {}
sentiments = {}
for comment, sentiment in zip(comments, results):
comment_sentiment = list(sentiment.keys())[0]
output.append(comment_sentiment)
sentiments[comment] = comment_sentiment
# подсчет результатов для каждой категории
distribution['positive'] = output.count('positive')
distribution['negative'] = output.count('negative')
distribution['neutral'] = output.count('neutral')
distribution['skip'] = output.count('skip')
distribution['speech'] = output.count('speech')
# вызов функции для создания txt-отчета
self.detailed_report(f_name, sentiments, distribution)
return distribution, sentiments
def detailed_report(self, group_name, sentiments, distribution):
""" Создание детального txt-отчета.
Вспомогательный метод для get_sentiment.
Не рекомендуется для использования в качестве самостоятельной функции.
"""
file = open(f'reports/{group_name}', 'w')
file.write(f'Тональность комментариев из сообщества {group_name.replace(".txt", "")}: \n')
negative = [k for k,v in sentiments.items() if v == 'negative']
positive = [k for k,v in sentiments.items() if v == 'positive']
neutral = [k for k,v in sentiments.items() if v == 'neutral']
skip = [k for k,v in sentiments.items() if v == 'skip']
speech = [k for k,v in sentiments.items() if v == 'speech']
file.write(f'\nНегативные комментарии: {distribution["negative"]}\n\n')
for comment in negative:
file.write(comment + '\n')
file.write(f'\nПозитивные комментарии: {distribution["positive"]}\n\n')
for comment in positive:
file.write(comment + '\n')
file.write(f'\nНейтральные комментарии: {distribution["neutral"]}\n\n')
for comment in neutral:
file.write(comment + '\n')
file.write(f'\nКомментарии без смысла: {distribution["skip"]}\n\n')
for comment in skip:
file.write(comment + '\n')
file.write(f'\nРечь, цитирование: {distribution["speech"]}\n\n')
for comment in speech:
file.write(comment + '\n')
file.close()
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
message = ['Волосы у девушки просто огонь!!! Красота!!!']
results = model.predict(message, k=len(message))
for message, sentiment in zip(message, results):
# привет -> {'speech': 1.0000100135803223, 'skip': 0.0020607432816177607}
# люблю тебя!! -> {'positive': 0.9886782765388489, 'skip': 0.005394937004894018}
# малолетние дебилы -> {'negative': 0.9525841474533081, 'neutral': 0.13661839067935944}]
print(message, '->', sentiment)
from sklearn.metrics import classification_report
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import StratifiedKFold
from sklearn.base import TransformerMixin
import nltk
from sklearn.model_selection import cross_val_predict
from sklearn.metrics import log_loss, accuracy_score
from gensim.models import Doc2Vec
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV
patterns = "[A-Za-z0-9!#$%&'()*+,./:;<=>?@[\]^_`{|}~—\"\-]+"
fasttext.FastText.eprint = lambda x: None
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer, lemmatize=True)
def csv_reader(file_obj):
"""
Read a csv file
"""
reader = csv.reader(file_obj)
data = []
for row in reader:
row = str(" ".join(row))
row = row.replace("#", "hashtag")
if row != "":
data.append(row)
data[
STOP_WORDS = get_stop_words("ru")
MIN_WORD_LEN = 3
FIGURE_SIZE = (20, 8)
# FONT_SIZE = 20
FONT_SIZE = 40
ascii_lowercase = set(ascii_lowercase)
digits = set(digits)
OBSCENE_WORDS_FILTER = ObsceneWordsFilter(conf.bad_words_re,
conf.good_words_re)
OBSCENE_FILTER_FALSES = ["команд", "дубляж"]
LETSGO_WORDS = {"пойдем", "пошли", "пойдемте", "погнали", "го", "хочет"}
SENTIMENT_TOKENIZER = RegexTokenizer()
SENTIMENT_MODEL = FastTextSocialNetworkModel(tokenizer=SENTIMENT_TOKENIZER)
def mode(a):
mode_value, mode_count = scipy_mode_fn(a)
return mode_value
def strip_message(msg):
msg = msg.strip()
msg = msg.replace('\n', ' ')
msg = msg.lower()
return msg
def __init__(self):
self.db = DB()
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
self.tokens = self.tokenizer.split('всё очень плохо')
import dostoevsky
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
#dostoevsky download fasttext-social-network-model
data = open("words.txt", "r")
words = data.readlines()
data.close()
file = open("estimations.txt", "w")
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
i = 0
while i < len(words):
results = model.predict(words[i:i + 100], k=1)
for word, sentiment in zip(words[i:i + 100], results):
estimation = list(sentiment.keys())[0]
if estimation == 'skip' or estimation == 'speech':
estimation = 'neutral'
if estimation == 'negative': estimation = '-1'
elif estimation == 'neutral': estimation = '0'
elif estimation == 'positive': estimation = '1'
file.write(word.split()[0] + ' ' + estimation + '\n')
i += 100
file.close()
def sentiment_analysis(text):
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(text, k=3)
return results
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
import pymorphy2
import re
import matplotlib.pyplot as plt
import numpy as np
import pylab
from matplotlib import mlab
# Импортируем пакет со вспомогательными функциями
from matplotlib import mlab
morph = pymorphy2.MorphAnalyzer()
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
tweets = [] # список со всеми не пустыми твиттами
tweets_time = [] # время в минутах
tweets_length = {} # длина твиттов и количество твиттов с такой длиной
tweets_score = [] # оценка каждого твитта по третьему правилу
words_score = {} # оценка положительное/нейтральное/отрицательное слово
adj_positive = {}
adj_negative = {}
tweet_score_1st_rule = {} # оценка каждого твитта по первому правилу
tweet_score_2nd_rule = {} # оценка каждого твитта по второму правилу
tweet_score_3rd_rule = {} # оценка каждого твитта по третьему правилу
tweet_score_4th_rule = {} # оценка каждого твитта по четвертому правилу
words_cr = {} # словарь со словом и частью его частью речи
words_frequency = {} # количество твиттов, в котором встречалось это слово
data = open('data.txt')
tweets_amount = 0
