class SentimentAnalyzer:
def __init__(self):
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
def _map_sentiment(self, sentiment):
if sentiment == "positive":
return 1
if sentiment == "negative":
return -1
else:
return 0
def sentiment_label_dataframe(self, df: DataFrame) -> DataFrame:
df['sentiment'] = None
for i, row in df.iterrows():
results = self.model.predict([row['text']], k=30)
sentiment = max(list(results[0].items()),
key=lambda prob: prob[1])[0]
sentiment = self._map_sentiment(sentiment)
df.at[i, 'sentiment'] = sentiment
return df
def sentiment_label_sentence(self, sentence: str):
results = self.model.predict([sentence], k=30)
sentiment = max(list(results[0].items()), key=lambda prob: prob[1])[0]
return self._map_sentiment(sentiment)
class DostN:
def __init__(self):
self.db = DB()
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
self.tokens = self.tokenizer.split('всё очень плохо')
def ready_msg(self, messages, nnID):
results = self.model.predict(messages, k=2)
sentiments = []
for message, sentiment in zip(messages, results):
sentiments.append(sentiment)
try:
pos = sentiments[0]['positive']
self.db.setPos(nnID, pos)
except Exception as e:
print(e, 22)
pos = 0
self.db.setPos(nnID, pos)
try:
neg = sentiments[0]['negative']
self.db.setNegative(nnID, neg)
except Exception as e:
print(e, 29)
neg = 0
self.db.setNegative(nnID, neg)
try:
ne = sentiments[0]['neutral']
self.db.setNeutral(nnID, ne)
except Exception as e:
print(e, 36)
ne = 0
self.db.setNeutral(nnID, ne)
return (sentiments)
def parse_senstiment(self, text):
tokenizer = RegexTokenizer()
tokens = tokenizer.split('всё очень плохо') # хз
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
return model.predict([text], k=2)[0]
def validation_comment(
self, film_id, user_id
): # валидация отзыва, если он проходит проверку то comm_check = 1
x = self.conn.execute(
'SELECT rating1, rating2, rating3, rating4, rating5, comm_text '
'FROM Comments WHERE film_id = (?) and user_id = (?)', (
film_id[0],
user_id,
)).fetchall()
sr = 0
for r in x[0][0:5]:
sr = sr + r
sr = sr / 5
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
result = model.predict([x[0][5]], k=5)
dl = result[0]['neutral'] + result[0]['negative'] + result[0][
'positive']
res = 10 * (result[0]['neutral'] / dl - result[0]['negative'] / dl +
result[0]['positive'] / dl)
if abs(sr - res) <= valid_const:
self.set_comment(film_id, user_id, 'comm_check', 1)
self.conn.commit()
self.set_comment(film_id, user_id, 'second_check', 0)
self.conn.commit()
def graph():
file = filedialog.askopenfilename(filetypes=(("Text files", "*.txt"),
("all files", "*.*")))
f = open(file)
raw = f.read()
sentences = nltk.sent_tokenize(raw)
command = 'download'
arguments = ['fasttext-social-network-model']
if command == 'download':
downloader = DataDownloader()
for filename in arguments:
if filename not in AVAILABLE_FILES:
raise ValueError(f'Unknown package: {filename}')
source, destination = AVAILABLE_FILES[filename]
destination_path: str = os.path.join(DATA_BASE_PATH, destination)
if os.path.exists(destination_path):
continue
downloader.download(source=source, destination=destination)
else:
raise ValueError('Unknown command')
import dostoevsky
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = sentences
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
positive_values_all = [
sentiment.get('positive')
for message, sentiment in zip(messages, results)
]
positive_values = [
0.0 if value == None else value for value in positive_values_all
]
negative_values_all = [
sentiment.get('negative')
for message, sentiment in zip(messages, results)
]
negative_values = [
0.0 if value == None else value for value in negative_values_all
]
summary = (len(negative_values))
n_value = np.array(negative_values)
p_value = np.array(positive_values)
counts_value = np.arange(summary)
plt.plot(counts_value, p_value, n_value)
plt.show()
def analyze(all_news):
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
texts = [news['title'] + ' ' + news['article'] for news in all_news]
results = model.predict(texts)
for i in range(len(all_news)):
all_news[i]['tone'] = results[i]
return all_news
def sentiment_analysis(data):
langid.set_languages(['en', 'ru'])
lang = langid.classify(data['text'][0])[0]
if lang == 'ru':
labels = data['from'].unique()
msg_df = data.loc[data.text != '']
messages_1 = list(msg_df.text[msg_df['from'] == labels[0]])
messages_2 = list(msg_df.text[msg_df['from'] == labels[1]])
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results_1 = model.predict(messages_1, k=2)
sentiments_1 = []
for sentiment in results_1:
# привет -> {'speech': 1.0000100135803223, 'skip': 0.0020607432816177607}
# люблю тебя!! -> {'positive': 0.9886782765388489, 'skip': 0.005394937004894018}
# малолетние дебилы -> {'negative': 0.9525841474533081, 'neutral': 0.13661839067935944}]
tone = 0
if 'positive' in sentiment:
tone += sentiment['positive']
if 'negative' in sentiment:
tone -= sentiment['negative']
sentiments_1.append(tone)
results_2 = model.predict(messages_2, k=2)
sentiments_2 = []
for sentiment in results_2:
tone = 0
if 'positive' in sentiment:
tone += sentiment['positive']
if 'negative' in sentiment:
tone -= sentiment['negative']
sentiments_2.append(tone)
return sentiments_1, sentiments_2
def analysis_data(messages):
res = []
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
print(messages)
i = 0
for message in messages:
data = model.predict([message], k=2)[0]
i += 1
for key, value in data.items():
if round(value, 1) == 1:
res.append(key)
if len(res) != i:
res.append('neutral')
return res
def get_sentiment(text):
if not text:
return 5
model = FastTextSocialNetworkModel(tokenizer=RegexTokenizer(),
lemmatize=True)
results = model.predict([text], k=2)
result = 0
result = result + (min(results[0]['positive'], 1)
if 'positive' in results[0] else 0)
result = result - (min(results[0]['negative'], 1)
if 'negative' in results[0] else 0)
return round((result + 1) * 5)
def text_analysis(all_comments):
texts = [comment['comment'] for comment in all_comments]
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(texts, k=5)
for i, sentiment in enumerate(results):
all_comments[i]['positive'] = sentiment['positive']
all_comments[i]['negative'] = sentiment['negative']
all_comments[i]['neutral'] = sentiment['neutral']
return all_comments
def test_tonality():
test = [
'''Ужасное расположение и распределение товаров. Два уровня и на каждом свои кассы. Чтобы купить разные группы товаров нужно отстоять две очереди.''',
'''Это не шаурма это ужас,куча майонеза,лук одна кожура верхний слой который мы при готовке выкладываем, картофель фри из пачек сухой, мясо порезано тонкими пластами, не пойму как оно приготавливалось явно не на гриле, мясо было не свежее, в итоге самый съедобный оказался лаваш, не рекомендую.''',
'''Рядом с домом, вкусная картошечка и обалденные молочные коктейли и довольно быстрое обслуживание, приятные кассиры''',
'''Замечательный телефон, пользуюсь им уже 2 года, очень нравится!''',
'''Был в этом магазине в прошлом году, больше туда приходить не собираюсь, некомпетентные продавцы, плохое обслуживание((''',
'''Идеальный мастер!:)''',
'''Уроды, как можно так поступить с человеком, просто ужас?!'''
]
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(test, k=2)
for i, el in enumerate(zip(test, results)):
token, sentiment = el
print_tonality(test[i], sentiment.items())
def tonal_analize(string):
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = [string]
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
negative = 0
positive = 0
if 'positive' in sentiment:
positive += sentiment['positive']
if 'negative' in sentiment:
negative += sentiment['negative']
delta = positive - negative
return delta
def return_sentiment(shcode):
response = requests.get(
get_url(f'https://www.instagram.com/p/{shcode}/?__a=1'))
data = json.loads(response.content)
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
sentim = []
edges = data['graphql']['shortcode_media']['edge_media_preview_comment'][
'edges']
for com in edges:
# print(com['node']['text'])
results = model.predict([com['node']['text']],
k=len(com['node']['text']))
for x in results:
sentim.append(list(x.keys())[0])
return sentim
def processSentiment(message, stage):
duration = '0.0'
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
phrases = []
for phrase in message:
duration = phrase['end_time']
for item in phrase['alternatives']:
phrases.append(item['transcript'])
results = model.predict(phrases, k=2)
for phrase, sentiments in zip(phrases, results):
for sentiment in sentiments.keys():
if sentiment not in ['neutral', 'skip']:
answer = sentiment
result = stages[stage][answer]
return [answer, result, float(duration[0:-1:])]
def get_sent(x):
def one_hot_encode_sent(x):
"""
(pos, neg, neu)
"""
if x[0] == 'positive':
return (1, 0, 0)
elif x[0] == 'negative':
return (0, 1, 0)
else:
return (0, 0, 1)
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(x, k=1)
results = [(list(r.keys())[0], list(r.values())[0]) for r in results]
results = list(map(one_hot_encode_sent, results))
results = [
results[i] if x[i] != 'EMPTY_TEXT' else (0, 0, 0)
for i in range(len(results))
]
return [pd.Series(x) for x in zip(*results)] # return three series
def main():
db_con = init_sync()
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
with db_con:
cur = db_con.cursor()
# Запрос к предложениям выдленным томитой
cur.execute("SELECT id, text FROM filtered WHERE text IS NOT NULL")
filtreds = [{x[0]: x[1]} for x in cur.fetchall() if len(x[1]) > 0]
tonality = []
for filtred in filtreds:
results = model.predict(filtred.values(), k=2)
for i, el in enumerate(zip(filtred.values(), results)):
token, sentiment = el
tonality.clear()
for key, value in filtred.items():
id_filtered = key
text = value
for item in sentiment.items():
tonality.append(item)
insert_tonality(db_con, id_filtered, tonality)
print_tonality(text, tonality)
def begin():
file = filedialog.askopenfilename(filetypes=(("Text files", "*.txt"),
("all files", "*.*")))
f = open(file)
raw = f.read()
sentences = nltk.sent_tokenize(raw)
command = 'download'
arguments = ['fasttext-social-network-model']
if command == 'download':
downloader = DataDownloader()
for filename in arguments:
if filename not in AVAILABLE_FILES:
raise ValueError(f'Unknown package: {filename}')
source, destination = AVAILABLE_FILES[filename]
destination_path: str = os.path.join(DATA_BASE_PATH, destination)
if os.path.exists(destination_path):
continue
downloader.download(source=source, destination=destination)
else:
raise ValueError('Unknown command')
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = sentences
results = model.predict(messages, k=2)
for message, sentiment in zip(messages, results):
analysis_line = '\n', message, '\n', '->', '\n', sentiment, '\n'
text.insert(END, analysis_line)
def sentiment_analysis(text):
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
results = model.predict(text, k=3)
return results
class FeaturesProcessor:
CATEGORY = 'category_id'
def __init__(self,
model_dir_path: str,
verbose=0,
use_markers=True,
use_morphology=True,
embed_model_stopwords=True,
use_w2v=True,
use_sentiment=True):
"""
:param str model_dir_path: path with all the models
:param int verbose: 0 for no logging, 1 for time counts logging and 2 for all warnings
:param bool embed_model_stopwords: do count stopwords during w2v vectorization
:param use_w2v: do w2v vectorization
:param use_sentiment: do sentiment extraction
"""
self._model_dir_path = model_dir_path
self._verbose = verbose
self._use_markers = use_markers
self._use_morphology = use_morphology
self._embed_model_stopwords = embed_model_stopwords
self._use_w2v = use_w2v
self._use_sentiment = use_sentiment
if self._verbose:
print("Processor initialization...\t", end="", flush=True)
self.relations_related = relations_related
self.stop_words = nltk.corpus.stopwords.words('russian')
self.count_words_x = count_words_x
self.count_words_y = count_words_y
self.pairs_words = pairs_words
self.vectorizer = pickle.load(open(os.path.join(model_dir_path, 'tf_idf', 'pipeline.pkl'), 'rb'))
# preprocessing functions
self._uppercased = lambda snippet, length: sum(
[word[0].isupper() if len(word) > 0 else False for word in snippet.split()]) / length
self._start_with_uppercase = lambda snippet, length: sum(
[word[0].isupper() if len(word) > 0 else False for word in snippet.split(' ')]) / length
if self._use_w2v:
self.embed_model_path = os.path.join(model_dir_path, 'w2v', 'default', 'model.vec')
self._synonyms_vocabulary = synonyms_vocabulary
# embeddings
if self.embed_model_path[-4:] in ['.vec', '.bin']:
self.word2vec_model = KeyedVectors.load_word2vec_format(self.embed_model_path,
binary=self.embed_model_path[-4:] == '.bin')
else:
self.word2vec_model = Word2Vec.load(self.embed_model_path)
test_word = ['дерево', 'NOUN']
try:
self.word2vec_vector_length = len(self.word2vec_model.wv.get_vector(test_word[0]))
self.word2vec_tag_required = False
except KeyError:
self.word2vec_vector_length = len(self.word2vec_model.wv.get_vector('_'.join(test_word)))
self.word2vec_tag_required = True
self.word2vec_stopwords = embed_model_stopwords
self._remove_stop_words = lambda lemmatized_snippet: [word for word in lemmatized_snippet if
word not in self.stop_words]
self.fpos_combinations = FPOS_COMBINATIONS
if self._use_sentiment:
self.sentiment_model = FastTextSocialNetworkModel(tokenizer=RegexTokenizer())
# self._context_length = 3
if self._verbose:
print('[DONE]')
def _find_y(self, snippet_x, snippet_y, loc_x):
result = self.annot_text.find(snippet_y, loc_x + len(snippet_x) - 1)
if result < 1:
result = self.annot_text.find(snippet_y, loc_x + 1)
if result < 1:
result = loc_x + 1
return result
def __call__(self, df_, annot_text, annot_tokens, annot_sentences, annot_lemma, annot_morph, annot_postag,
annot_syntax_dep_tree):
df = df_[:]
df.snippet_x = df.snippet_x.replace('\n', ' ', regex=True).replace(' ', ' ', regex=True)
df.snippet_y = df.snippet_y.replace('\n', ' ', regex=True).replace(' ', ' ', regex=True)
self.annot_text = annot_text.replace('\n', ' ').replace(' ', ' ')
self.annot_tokens = annot_tokens
self.annot_sentences = annot_sentences
self.annot_lemma = annot_lemma
self.annot_morph = annot_morph
self.annot_postag = annot_postag
self.annot_syntax_dep_tree = annot_syntax_dep_tree
t, t_final = None, None
if self._verbose:
t = time.time()
t_final = t
print('1\t', end="", flush=True)
df['is_broken'] = False
# map discourse units to annotations
if not 'loc_x' in df.keys():
df['loc_x'] = df.snippet_x.map(self.annot_text.find)
if not 'loc_y' in df.keys():
df['loc_y'] = df.apply(lambda row: self._find_y(row.snippet_x, row.snippet_y, row.loc_x - 1), axis=1)
df['token_begin_x'] = df.loc_x.map(self.locate_token)
df['token_begin_y'] = df.loc_y.map(self.locate_token)
try:
df['token_end_y'] = df.apply(lambda row: self.locate_token(row.loc_y + len(row.snippet_y)), axis=1) # -1
df['token_end_y'] = df['token_end_y'] + (df['token_end_y'] == df['token_begin_y']) * 1
except:
if self._verbose == 2:
print(f'Unable to locate second snippet >>> {(df.snippet_x.values, df.snippet_y.values)}',
file=sys.stderr)
df['tokens_x'] = df.snippet_x.map(lambda row: row.split())
df['tokens_y'] = df.snippet_y.map(lambda row: row.split())
# df['left_context'] = ['_END_'] * self._context_length
# df['right_context'] = ['_END_'] * self._context_length
df['same_sentence'] = 0
df['is_broken'] = True
return df
# length of tokens sequence
df['len_w_x'] = df['token_begin_y'] - df['token_begin_x']
df['len_w_y'] = df['token_end_y'] - df['token_begin_y'] # +1
df['snippet_x_locs'] = df.apply(lambda row: [[pair for pair in [self.token_to_sent_word(token) for token in
range(row.token_begin_x, row.token_begin_y)]]],
axis=1)
df['snippet_x_locs'] = df.snippet_x_locs.map(lambda row: row[0])
# print(df[['snippet_x', 'snippet_y', 'snippet_x_locs']].values)
broken_pair = df[df.snippet_x_locs.map(len) < 1]
if not broken_pair.empty:
print(
f"Unable to locate first snippet >>> {df[df.snippet_x_locs.map(len) < 1][['snippet_x', 'snippet_y', 'token_begin_x', 'token_begin_y', 'loc_x', 'loc_y']].values}",
file=sys.stderr)
df = df[df.snippet_x_locs.map(len) > 0]
# print(df[['snippet_x', 'snippet_y', 'token_begin_y', 'token_end_y']])
df['snippet_y_locs'] = df.apply(lambda row: [
[pair for pair in [self.token_to_sent_word(token) for token in range(row.token_begin_y, row.token_end_y)]]],
axis=1)
df['snippet_y_locs'] = df.snippet_y_locs.map(lambda row: row[0])
broken_pair = df[df.snippet_y_locs.map(len) < 1]
if not broken_pair.empty:
print(
f"Unable to locate second snippet >>> {df[df.snippet_y_locs.map(len) < 1][['snippet_x', 'snippet_y', 'token_begin_x', 'token_begin_y', 'token_end_y', 'loc_x', 'loc_y']].values}",
file=sys.stderr)
df2 = df[df.snippet_y_locs.map(len) < 1]
_df2 = pd.DataFrame({
'snippet_x': df2['snippet_y'].values,
'snippet_y': df2['snippet_x'].values,
'loc_y': df2['loc_x'].values,
'token_begin_y': df2['token_begin_x'].values,
})
df2 = _df2[:]
df2['loc_x'] = df2.apply(lambda row: self.annot_text.find(row.snippet_x, row.loc_y - 3), axis=1)
df2['token_begin_x'] = df2.loc_x.map(self.locate_token)
# df2['loc_y'] = df2.apply(lambda row: self._find_y(row.snippet_x, row.snippet_y, row.loc_x), axis=1)
df2['token_end_y'] = df2.apply(lambda row: self.locate_token(row.loc_y + len(row.snippet_y)), # + 1,
axis=1) # -1
# df2['token_begin_x'] = df2['token_begin_y']
# df2['token_begin_y'] = df2.loc_y.map(self.locate_token)
df2['len_w_x'] = df2['token_begin_y'] - df2['token_begin_x']
df2['len_w_y'] = df2['token_end_y'] - df2['token_begin_y'] # +1
df2['snippet_x_locs'] = df2.apply(
lambda row: [[pair for pair in [self.token_to_sent_word(token) for token in
range(row.token_begin_x, row.token_begin_y)]]], axis=1)
df2['snippet_x_locs'] = df2.snippet_x_locs.map(lambda row: row[0])
df2['snippet_y_locs'] = df2.apply(
lambda row: [[pair for pair in [self.token_to_sent_word(token) for token in
range(row.token_begin_y, row.token_end_y)]]], axis=1)
df2['snippet_y_locs'] = df2.snippet_y_locs.map(lambda row: row[0])
broken_pair = df2[df2.snippet_y_locs.map(len) < 1]
if not broken_pair.empty:
print(
f"Unable to locate second snippet AGAIN >>> {df2[df2.snippet_y_locs.map(len) < 1][['snippet_x', 'snippet_y', 'token_begin_x', 'token_begin_y', 'token_end_y', 'loc_x', 'loc_y']].values}",
file=sys.stderr)
df = df[df.snippet_y_locs.map(len) > 0]
df2 = df2[df2.snippet_x_locs.map(len) > 0]
df = pd.concat([df, df2])
# print(df[['snippet_x', 'snippet_y', 'snippet_y_locs', 'loc_x', 'loc_y']].values)
df.drop(columns=['loc_x', 'loc_y'], inplace=True)
if self._verbose:
print(time.time() - t)
t = time.time()
print('2\t', end="", flush=True)
# define a number of sentences and whether x and y are in the same sentence
df['sentence_begin_x'] = df.snippet_x_locs.map(lambda row: row[0][0])
df['sentence_begin_y'] = df.snippet_y_locs.map(lambda row: row[0][0])
df['sentence_end_y'] = df.snippet_y_locs.map(lambda row: row[-1][0])
df['number_sents_x'] = (df['sentence_begin_y'] - df['sentence_begin_x']) | 1
df['number_sents_y'] = (df['sentence_end_y'] - df['sentence_begin_y']) | 1
df['same_sentence'] = (df['sentence_begin_x'] == df['sentence_begin_y']).astype(int)
df['same_paragraph'] = df.apply(
lambda row: annot_text.find('\n', row.sentence_begin_x, row.sentence_end_y) != -1,
axis=1).astype(int)
df['same_paragraph'] = df['same_sentence'] | df['same_paragraph']
# find the common syntax root of x and y
df['common_root'] = df.apply(lambda row: [self.locate_root(row)], axis=1)
# find its relative position in text
# df['common_root_position'] = df.common_root.map(lambda row: self.map_to_token(row[0])) / len(annot_tokens)
# define its fPOS
# df['common_root_fpos'] = df.common_root.map(lambda row: self.get_postag(row)[0])
# 1 if it is located in y
df['root_in_y'] = df.apply(
lambda row: self.map_to_token(row.common_root[0]) > row.token_begin_y, axis=1).astype(int)
df.drop(columns=['common_root'], inplace=True)
if self._verbose:
print(time.time() - t)
t = time.time()
print('3\t', end="", flush=True)
# find certain markers for various relations
if self._use_markers:
for relation in self.relations_related:
df[relation + '_count' + '_x'] = df.snippet_x.map(lambda row: self._relation_score(relation, row))
df[relation + '_count' + '_y'] = df.snippet_y.map(lambda row: self._relation_score(relation, row))
if self._verbose:
print(time.time() - t)
t = time.time()
print('4\t', end="", flush=True)
# get tokens
df['tokens_x'] = df.apply(lambda row: self.get_tokens(row.token_begin_x, row.token_begin_y), axis=1)
df['tokens_x'] = df.apply(lambda row: row.tokens_x if len(row.tokens_x) > 0 else row.snippet_x.split(), axis=1)
df['tokens_y'] = df.apply(lambda row: self.get_tokens(row.token_begin_y, row.token_end_y - 1), axis=1)
df['tokens_y'] = df.apply(lambda row: row.tokens_y if len(row.tokens_y) > 0 else row.snippet_y.split(), axis=1)
# get lemmas
df['lemmas_x'] = df.snippet_x_locs.map(self.get_lemma)
df['lemmas_y'] = df.snippet_y_locs.map(self.get_lemma)
if self._verbose:
print(time.time() - t)
t = time.time()
print('5\t', end="", flush=True)
# ratio of uppercased words
df['upper_x'] = df.tokens_x.map(lambda row: sum(token.isupper() for token in row) / (len(row) + 1e-5))
df['upper_y'] = df.tokens_y.map(lambda row: sum(token.isupper() for token in row) / (len(row) + 1e-5))
# ratio of the words starting with upper case
df['st_up_x'] = df.tokens_x.map(lambda row: sum(token[0].isupper() for token in row) / (len(row) + 1e-5))
df['st_up_y'] = df.tokens_y.map(lambda row: sum(token[0].isupper() for token in row) / (len(row) + 1e-5))
# whether DU starts with upper case
df['du_st_up_x'] = df.tokens_x.map(lambda row: row[0][0].isupper()).astype(int)
df['du_st_up_y'] = df.tokens_y.map(lambda row: row[0][0].isupper()).astype(int)
if self._verbose:
print(time.time() - t)
t = time.time()
print('6\t', end="", flush=True)
# get morphology
if self._use_morphology:
df['morph_x'] = df.snippet_x_locs.map(self.get_morph)
df['morph_y'] = df.snippet_y_locs.map(self.get_morph)
# count presence and/or quantity of various language features in the whole DUs and at the beginning/end of them
df = df.apply(lambda row: self._linguistic_features(row, tags=MORPH_FEATS), axis=1)
df = df.apply(lambda row: self._first_and_last_pair(row), axis=1)
if self._verbose:
print(time.time() - t)
t = time.time()
print('7\t', end="", flush=True)
# count various vectors similarity metrics for morphology
if self._use_morphology:
linknames_for_snippet_x = df[[name + '_x' for name in MORPH_FEATS]]
linknames_for_snippet_y = df[[name + '_y' for name in MORPH_FEATS]]
df.reset_index(inplace=True)
df['morph_vec_x'] = pd.Series(self.columns_to_vectors_(linknames_for_snippet_x))
df['morph_vec_y'] = pd.Series(self.columns_to_vectors_(linknames_for_snippet_y))
df['morph_correlation'] = df[['morph_vec_x', 'morph_vec_y']].apply(
lambda row: spatial.distance.correlation(*row), axis=1)
df['morph_hamming'] = df[['morph_vec_x', 'morph_vec_y']].apply(lambda row: spatial.distance.hamming(*row),
axis=1)
df['morph_matching'] = df[['morph_vec_x', 'morph_vec_y']].apply(
lambda row: self.get_match_between_vectors_(*row), axis=1)
df.set_index('index', drop=True, inplace=True)
df = df.drop(columns=['morph_vec_x', 'morph_vec_y'])
if self._verbose:
print(time.time() - t)
t = time.time()
print('8\t', end="", flush=True)
# detect discourse markers
if self._use_markers:
for word in self.count_words_x:
df[word + '_count' + '_x'] = df.snippet_x.map(lambda row: self.count_marker_(word, row))
for word in self.count_words_y:
df[word + '_count' + '_y'] = df.snippet_y.map(lambda row: self.count_marker_(word, row))
# count stop words in the texts
df['stopwords_x'] = df.lemmas_x.map(self._count_stop_words)
df['stopwords_y'] = df.lemmas_y.map(self._count_stop_words)
if self._verbose:
print(time.time() - t)
t = time.time()
print('9\t', end="", flush=True)
# dummy function needed for self.vectorizer (do NOT remove)
def dummy(x):
return x
df.reset_index(drop=True, inplace=True)
tf_idf_x = self.vectorizer.transform(df['tokens_x'].map(lambda row: [_token.lower() for _token in row]))
tf_idf_y = self.vectorizer.transform(df['tokens_y'].map(lambda row: [_token.lower() for _token in row]))
df['cos_tf_idf_dist'] = paired_cosine_distances(tf_idf_x, tf_idf_y)
df['ang_cos_tf_idf_sim'] = 1. - np.arccos(df['cos_tf_idf_dist']) * 2. / np.pi
tf_idf_x = pd.DataFrame(tf_idf_x).add_prefix('tf_idf_x_')
tf_idf_y = pd.DataFrame(tf_idf_y).add_prefix('tf_idf_y_')
df = pd.concat([df, tf_idf_x, tf_idf_y], axis=1)
if self._verbose:
print(time.time() - t)
t = time.time()
print('10\t', end="", flush=True)
# count lexical similarity
df['bleu'] = df.apply(lambda row: self.get_bleu_score(row.lemmas_x, row.lemmas_y), axis=1)
if self._verbose:
print(time.time() - t)
t = time.time()
print('11\t', end="", flush=True)
if self._use_w2v:
# get average vector for each text
df = self._get_vectors(df)
if self._verbose:
print(time.time() - t)
t = time.time()
print('12\t', end="", flush=True)
# get relative positions in text
df['token_begin_x'] = df['token_begin_x'] / len(annot_tokens)
df['token_begin_y'] = df['token_begin_y'] / len(annot_tokens)
df['token_end_y'] = df['token_end_y'] / len(annot_tokens)
df['sentence_begin_x'] = df['sentence_begin_x'] / len(annot_sentences)
df['sentence_begin_y'] = df['sentence_begin_y'] / len(annot_sentences)
df['sentence_end_y'] = df['sentence_end_y'] / len(annot_sentences)
df['snippet_x_tmp'] = df.lemmas_x.map(lambda lemmas: ' '.join(lemmas).strip())
df['snippet_y_tmp'] = df.lemmas_y.map(lambda lemmas: ' '.join(lemmas).strip())
df['postags_x'] = df.snippet_x_locs.map(self.get_postags)
df['postags_y'] = df.snippet_y_locs.map(self.get_postags)
# count sentiments
if self._use_sentiment:
df = self._get_sentiments(df)
df = df.drop(columns=[
'lemmas_x', 'lemmas_y',
'snippet_x_locs', 'snippet_y_locs',
])
if self._use_morphology:
df = df.drop(columns=[
'morph_x', 'morph_y',
])
if self._verbose:
print(time.time() - t)
print('[DONE]')
print('estimated time:', time.time() - t_final)
return df.fillna(0.)
def locate_token(self, start):
for i, token in enumerate(self.annot_tokens):
if token.begin >= start:
return i
return i
def map_to_token(self, pair):
if pair == -1:
return -1
sentence, word = pair
if type(word) == list and len(word) == 1:
word = word[0]
return self.annot_sentences[sentence].begin + word
def token_to_sent_word(self, token):
for i, sentence in enumerate(self.annot_sentences):
if sentence.begin <= token < sentence.end:
return i, token - sentence.begin
for i, sentence in enumerate(self.annot_sentences):
if sentence.begin <= token + 1 < sentence.end:
return i, token - sentence.begin
return -1, -1
def locate_root(self, row):
if row.same_sentence:
for i, wordsynt in enumerate(self.annot_syntax_dep_tree[row.sentence_begin_x]):
if wordsynt.parent == -1:
return row.sentence_begin_x, [i]
return -1
def get_roots(self, locations):
res = []
for word in locations:
parent = self.annot_syntax_dep_tree[word[0]][word[1][0]].parent
if parent == -1:
res.append(word)
return res
def locate_attached(self, row):
res = []
sent_begin = self.annot_sentences[row.sentence_begin_x].begin
for i, wordsynt in enumerate(self.annot_syntax_dep_tree[row.sentence_begin_x]):
if row.token_begin_x - sent_begin <= i < row.token_end_y - sent_begin:
if wordsynt.parent == -1:
res.append(i)
return res
def get_tokens(self, begin, end):
return [self.annot_tokens[i].text for i in range(begin, end)]
def get_lemma(self, positions):
return [self.annot_lemma[position[0]][position[1]] for position in positions]
def get_postag(self, positions):
if positions:
if positions[0] == -1:
return ['']
result = [self.annot_postag[position[0]][position[1]] for position in positions]
if not result:
return ['X']
return result
return ['']
def get_morph(self, positions):
return [self.annot_morph[position[0]][position[1]] for position in positions]
def columns_to_vectors_(self, columns):
return [row + 1e-05 for row in np.array(columns.values.tolist())]
def get_match_between_vectors_(self, vector1, vector2):
return spatial.distance.hamming([k > 0.01 for k in vector1], [k > 0.01 for k in vector2])
def _get_fpos_vectors(self, row):
result = {}
for header in ['VERB', 'NOUN', '', 'ADV', 'ADJ', 'ADP', 'CONJ', 'PART', 'PRON' 'NUM']:
result[header + '_common_root'] = int(row.common_root_fpos == header)
for header in ['VERB', '', 'NOUN', 'ADJ', 'ADV', 'ADP', 'CONJ', 'PRON', 'PART', 'NUM', 'INTJ']:
result[header + '_common_root_att'] = int(row.common_root_att == header)
return row.append(pd.Series(list(result.values()), index=list(result.keys())))
@functools.lru_cache(maxsize=2048)
def count_marker_(self, word, row):
return bool(re.match(word, row, re.IGNORECASE))
@functools.lru_cache(maxsize=2048)
def locate_marker_(self, word, row):
for m in re.finditer(word, row):
index = m.start()
return (index + 1.) / len(row) * 100.
return -1.
def _svd_tfidf_matrix(self, matrix):
svd = TruncatedSVD(n_components=300)
return svd.fit_transform(matrix)
def _linguistic_features(self, row, tags):
""" Count occurences of each feature from MORPH_FEATS and/or SYNTAX_LINKS """
tags = MORPH_FEATS
def get_tags_for_snippet(morph_annot, mark='_x'):
result = dict.fromkeys(['%s%s' % (tag, mark) for tag in tags], 0)
for record in morph_annot:
for key, value in record.items():
try:
result['%s_%s%s' % (key, value, mark)] += 1
except KeyError as e:
if self._verbose == 2:
print(f"::: Did not find such key in MORPH_FEATS: {e} :::", file=sys.stderr)
else:
pass
return result
tags_for_snippet_x = get_tags_for_snippet(row.morph_x, '_x')
tags_for_snippet_y = get_tags_for_snippet(row.morph_y, '_y')
tags = dict(tags_for_snippet_x, **tags_for_snippet_y)
return row.append(pd.Series(list(tags.values()), index=list(tags.keys())))
def _count_stop_words(self, lemmatized_text, threshold=0):
return len([1 for token in lemmatized_text if len(token) >= threshold and token in self.stop_words])
def _relation_score(self, relation, row):
return sum([1 for value in self.relations_related[relation] if value in row])
def _postag(self, location):
return self.annot_postag[location[0]][location[1]]
def get_postags(self, locations):
result = []
for location in locations:
result.append(self._postag(location))
return ' '.join(result)
def _first_postags(self, locations, n=2):
result = []
for location in locations[:n]:
sent, word = location[0], location[1]
postag = self.annot_postag[sent][word]
result.append(postag)
return result
def _last_postags(self, locations, n=2):
result = []
for location in locations[-n:]:
sent, word = location[0], location[1]
postag = self.annot_postag[sent][word]
result.append(postag)
return result
def _first_and_last_pair(self, row):
def get_features_for_snippet(first_pair_text, first_pair_morph, last_pair_text, last_pair_morph, mark='_x'):
result = {}
for pos_combination in self.fpos_combinations:
result['first_' + pos_combination + mark] = int(pos_combination == first_pair_morph)
result['last_' + pos_combination + mark] = int(pos_combination == last_pair_morph)
for key in self.pairs_words:
if mark == key[-2:]:
if key[:-2] == 'first_pair':
for word in self.pairs_words[key]:
result[key[:-1] + word + mark] = int(bool(re.findall(word, first_pair_text, re.IGNORECASE)))
else:
for word in self.pairs_words[key]:
result[key[:-1] + word + mark] = int(bool(re.findall(word, last_pair_text, re.IGNORECASE)))
return result
# snippet X
first_pair_text_x = ' '.join([token for token in row.tokens_x[:2]]).lower()
first_pair_morph_x = '_'.join(self._first_postags(row.snippet_x_locs))
if len(row.tokens_x) > 2:
last_pair_text_x = ' '.join([token for token in row.tokens_x[-2:]]).lower()
last_pair_morph_x = '_'.join(self._last_postags(row.snippet_x_locs))
else:
last_pair_text_x = ' '
last_pair_morph_x = 'X'
features_of_snippet_x = get_features_for_snippet(first_pair_text_x, first_pair_morph_x,
last_pair_text_x, last_pair_morph_x,
'_x')
# snippet Y
first_pair_text_y = ' '.join([token for token in row.tokens_y[:2]]).lower()
first_pair_morph_y = '_'.join(self._first_postags(row.snippet_y_locs))
if len(row.tokens_y) > 2:
last_pair_text_y = ' '.join([token for token in row.tokens_y[-2:]]).lower()
last_pair_morph_y = '_'.join(self._last_postags(row.snippet_y_locs))
else:
last_pair_text_y = ' '
last_pair_morph_y = 'X'
features_of_snippet_y = get_features_for_snippet(first_pair_text_y, first_pair_morph_y,
last_pair_text_y, last_pair_morph_y,
'_y')
tags = dict(features_of_snippet_x, **features_of_snippet_y)
return row.append(pd.Series(list(tags.values()), index=list(tags.keys())))
def get_jaccard_sim(self, text1, text2):
txt1 = set(text1)
txt2 = set(text2)
c = len(txt1.intersection(txt2))
return float(c) / (len(txt1) + len(txt2) - c + 1e-05)
def get_bleu_score(self, text1, text2):
return bleu_score.sentence_bleu([text1], text2, weights=(0.5,))
def get_chrf_score(self, text1, text2):
try:
return chrf_score.corpus_chrf([text1], [text2], min_len=2)
except ZeroDivisionError:
return 0.
def _tag_postags_morph(self, locations):
result = []
for location in locations:
sent, word = location[0], location[1]
_postag = self.annot_morph[sent][word].get('fPOS')
if _postag:
result.append(self.annot_lemma[sent][word] + '_' + _postag)
else:
result.append(self.annot_lemma[sent][word])
return result
def _get_vectors(self, df):
def mean_vector(lemmatized_text):
res = list([np.zeros(self.word2vec_vector_length), ])
for word in lemmatized_text:
try:
res.append(self.word2vec_model[word])
except KeyError:
second_candidate = self._synonyms_vocabulary.get(word)
if second_candidate:
res.append(self.word2vec_model[second_candidate])
elif self.word2vec_stopwords and ('NOUN' in word or 'VERB' in word):
# print(f'There is no "{word}" in vocabulary of the given model; ommited', file=sys.stderr)
pass
mean = sum(np.array(res)) / (len(res) - 1 + 1e-25)
return mean
if not self.word2vec_stopwords:
df.lemmas_x = df.lemmas_x.map(self._remove_stop_words)
df.lemmas_y = df.lemmas_y.map(self._remove_stop_words)
# Add the required UPoS postags (as in the rusvectores word2vec model's vocabulary)
if self.word2vec_tag_required:
df.lemmas_x = df.snippet_x_locs.map(self._tag_postags_morph)
df.lemmas_y = df.snippet_y_locs.map(self._tag_postags_morph)
# Make two dataframes with average vectors for x and y,
# merge them with the original dataframe
df_embed_x = df.lemmas_x.apply(mean_vector).values.tolist()
df_embed_y = df.lemmas_y.apply(mean_vector).values.tolist()
embeddings = pd.DataFrame(df_embed_x).merge(pd.DataFrame(df_embed_y), left_index=True, right_index=True)
embeddings['cos_embed_dist'] = paired_cosine_distances(df_embed_x, df_embed_y)
embeddings['eucl_embed_dist'] = paired_euclidean_distances(df_embed_x, df_embed_y)
df = pd.concat([df.reset_index(drop=True), embeddings.reset_index(drop=True)], axis=1)
return df
def _get_sentiments(self, df):
try:
temp = df.snippet_x.map(lambda row: self.sentiment_model.predict([row]))
except:
temp = df.snippet_x.map(lambda row: [{'positive': 0., 'negative': 0.}])
for key in ['positive', 'negative']:
df['sm_x_' + key] = temp.map(lambda row: row[0].get(key))
try:
temp = df.snippet_y.map(lambda row: self.sentiment_model.predict([row]))
except:
temp = df.snippet_y.map(lambda row: [{'positive': 0., 'negative': 0.}])
for key in ['positive', 'negative']:
df['sm_y_' + key] = temp.map(lambda row: row[0].get(key))
return df
class CommentsResearcher:
""" Класс для анализа тональности комментариев """
def __init__(self):
self.tokenizer = RegexTokenizer()
self.model = FastTextSocialNetworkModel(tokenizer=self.tokenizer)
def get_sentiment(self, file):
""" Опредение тональности комментариев
Возвращает 2 значения:
- distribution (dict)
словарь в количеством комментариев в разных категориях
- sentiments (dict)
словарь с тональностью каждого комментария
Подробная информация о результатах записывается в файл с именем сообщества.
Файл сохраняется в папку 'reports'.
Аргументы:
file (_io.TextIOWrapper): текстовой файл с комментариями (разделитель \t)
"""
comments = file.read().split('\t')
f_name = os.path.basename(file.name)
# получение тональности для комментариев с помощью модели
results = self.model.predict(comments[0:-1], k=1)
output = []
distribution = {}
sentiments = {}
for comment, sentiment in zip(comments, results):
comment_sentiment = list(sentiment.keys())[0]
output.append(comment_sentiment)
sentiments[comment] = comment_sentiment
# подсчет результатов для каждой категории
distribution['positive'] = output.count('positive')
distribution['negative'] = output.count('negative')
distribution['neutral'] = output.count('neutral')
distribution['skip'] = output.count('skip')
distribution['speech'] = output.count('speech')
# вызов функции для создания txt-отчета
self.detailed_report(f_name, sentiments, distribution)
return distribution, sentiments
def detailed_report(self, group_name, sentiments, distribution):
""" Создание детального txt-отчета.
Вспомогательный метод для get_sentiment.
Не рекомендуется для использования в качестве самостоятельной функции.
"""
file = open(f'reports/{group_name}', 'w')
file.write(f'Тональность комментариев из сообщества {group_name.replace(".txt", "")}: \n')
negative = [k for k,v in sentiments.items() if v == 'negative']
positive = [k for k,v in sentiments.items() if v == 'positive']
neutral = [k for k,v in sentiments.items() if v == 'neutral']
skip = [k for k,v in sentiments.items() if v == 'skip']
speech = [k for k,v in sentiments.items() if v == 'speech']
file.write(f'\nНегативные комментарии: {distribution["negative"]}\n\n')
for comment in negative:
file.write(comment + '\n')
file.write(f'\nПозитивные комментарии: {distribution["positive"]}\n\n')
for comment in positive:
file.write(comment + '\n')
file.write(f'\nНейтральные комментарии: {distribution["neutral"]}\n\n')
for comment in neutral:
file.write(comment + '\n')
file.write(f'\nКомментарии без смысла: {distribution["skip"]}\n\n')
for comment in skip:
file.write(comment + '\n')
file.write(f'\nРечь, цитирование: {distribution["speech"]}\n\n')
for comment in speech:
file.write(comment + '\n')
file.close()
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
tokens = tokenizer.split(
'всё очень плохо') # [('всё', None), ('очень', None), ('плохо', None)]
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
message = ['Волосы у девушки просто огонь!!! Красота!!!']
results = model.predict(message, k=len(message))
for message, sentiment in zip(message, results):
# привет -> {'speech': 1.0000100135803223, 'skip': 0.0020607432816177607}
# люблю тебя!! -> {'positive': 0.9886782765388489, 'skip': 0.005394937004894018}
# малолетние дебилы -> {'negative': 0.9525841474533081, 'neutral': 0.13661839067935944}]
print(message, '->', sentiment)
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
#dostoevsky download fasttext-social-network-model
data = open("words.txt", "r")
words = data.readlines()
data.close()
file = open("estimations.txt", "w")
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
i = 0
while i < len(words):
results = model.predict(words[i:i + 100], k=1)
for word, sentiment in zip(words[i:i + 100], results):
estimation = list(sentiment.keys())[0]
if estimation == 'skip' or estimation == 'speech':
estimation = 'neutral'
if estimation == 'negative': estimation = '-1'
elif estimation == 'neutral': estimation = '0'
elif estimation == 'positive': estimation = '1'
file.write(word.split()[0] + ' ' + estimation + '\n')
i += 100
file.close()
# freq.write(new_string)
# freq.close()
#
# # Аналогичным образом необходимо записать информацию о длине (числе слов) твитов в файл twits_length.txt:
# tweets_length = list(tweets_length.items())
# tweets_length.sort(key=lambda i: i[1], reverse=True)
# length = open("twits_length.txt", 'w')
# for i in tweets_length:
# new_string = str(i[0]) + ' - ' + str(i[1]) + ' - ' + str(round(i[1]*100/tweets_amount, 2)) + '%' + '\n'
# length.write(new_string)
# length.close()
# конец 1
#
# Далее необходимо создать файл с Вашей личной оценкой по каждому из слов в списке frequency.txt
est = open("estimations.txt", 'w')
results = model.predict(list(words_score.keys()))
for message, sentiment in zip(list(words_score.keys()), results):
# if words_frequency[message] < 100: #для наиболее встречаемых слов ввожу значение самостоятельно
score = list(sentiment.keys())[0]
if score == 'neutral':
score = 0
words_score[message] = score
elif score == 'positive':
score = 1
words_score[message] = score
if words_cr[message] == 'ADJF':
adj_positive[message] = words_frequency[message]
else:
score = -1
words_score[message] = score
if words_cr[message] == 'ADJF':
plt.ylabel('Counts')
df['Analysis'].value_counts().plot(kind = 'bar')
plt.show()
!pip3 install dostoevsky
!python3 -m dostoevsky download fasttext-social-network-model
from dostoevsky.tokenization import RegexTokenizer
from dostoevsky.models import FastTextSocialNetworkModel
tokenizer = RegexTokenizer()
model = FastTextSocialNetworkModel(tokenizer=tokenizer)
messages = df['Tweets'].apply(cleanTxt)
results = model.predict(messages, k=2)
message_skippy = []
message_negative = []
message_positive = []
message_speech = []
for message, sentiment in zip(messages, results):
senti = max(sentiment)
if senti == 'skip':
message_skippy.append(message)
if senti == 'negative':
message_negative.append(message)
if senti == 'positive':
message_positive.append(message)
if senti == 'speech':
message_speech.append(message)
