forked from Charcoal-SE/SmokeDetector
-
Notifications
You must be signed in to change notification settings - Fork 0
/
spamhandling.py
150 lines (135 loc) · 6.41 KB
/
spamhandling.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
import json
import sys
import time
from threading import Thread
from findspam import FindSpam
from datahandling import *
from parsing import get_user_from_url, unescape_title,\
escape_special_chars_in_title, to_protocol_relative
from bayesianfuncs import bayesian_score
from globalvars import GlobalVars
from datetime import datetime
from parsing import url_to_shortlink
from metasmoke import Metasmoke
def should_whitelist_prevent_alert(user_url, reasons):
is_whitelisted = is_whitelisted_user(get_user_from_url(user_url))
if not is_whitelisted:
return False
reasons_copy = list(set(reasons))
reasons_copy_copy = list(reasons_copy)
for reason in reasons_copy:
if "username" in reason:
reasons_copy_copy.remove(reason)
return len(reasons_copy_copy) == 0
def check_if_spam(title, body, user_name, user_url, post_site, post_id, is_answer, body_is_summary):
if not body:
body = ""
test, why = FindSpam.test_post(title, body, user_name, post_site, is_answer, body_is_summary)
if is_blacklisted_user(get_user_from_url(user_url)):
test.append("Blacklisted user")
blacklisted_user_data = get_blacklisted_user_data(get_user_from_url(user_url))
if len(blacklisted_user_data) > 1:
message_url = blacklisted_user_data[1]
blacklisted_post_url = blacklisted_user_data[2]
if blacklisted_post_url:
why += u"Blacklisted user - blacklisted for {} by {}\n".format(blacklisted_post_url, message_url)
else:
why += u"Blacklisted user - blacklisted by {}\n".format(message_url)
if 0 < len(test):
if has_already_been_posted(post_site, post_id, title) or is_false_positive((post_id, post_site)) \
or should_whitelist_prevent_alert(user_url, test) \
or is_ignored_post((post_id, post_site)) \
or is_auto_ignored_post((post_id, post_site)):
return False, None, "" # Don't repost. Reddit will hate you.
return True, test, why
return False, None, ""
def check_if_spam_json(data):
d = json.loads(json.loads(data)["data"])
try:
_ = d["ownerUrl"] # noqa
except:
# owner's account doesn't exist anymore, no need to post it in chat:
# http://chat.stackexchange.com/transcript/message/18380776#18380776
return False, None, ""
title = d["titleEncodedFancy"]
title = unescape_title(title)
body = d["bodySummary"]
poster = d["ownerDisplayName"]
url = d["url"]
post_id = str(d["id"])
print time.strftime("%Y-%m-%d %H:%M:%S"), title.encode("ascii", errors="replace")
quality_score = bayesian_score(title)
print quality_score
if quality_score < 0.3 and d["siteBaseHostAddress"] == "stackoverflow.com":
print GlobalVars.bayesian_testroom.send_message("[ SmokeDetector | BayesianBeta ] Quality score " + str(quality_score * 100) + ": [" + title + "](" + url + ")")
site = d["siteBaseHostAddress"]
site = site.encode("ascii", errors="replace")
sys.stdout.flush()
is_spam, reason, why = check_if_spam(title, body, poster, url, site, post_id, False, True)
return is_spam, reason, why
def handle_spam(title, body, poster, site, post_url, poster_url, post_id, reasons, is_answer, why=""):
post_url = to_protocol_relative(url_to_shortlink(post_url))
poster_url = to_protocol_relative(poster_url)
reason = ", ".join(reasons).capitalize()
if "Url" in reason:
reason = reason.replace("Url", "URL")
append_to_latest_questions(site, post_id, title if not is_answer else "")
if len(reasons) == 1 and ("All-caps title" in reasons or
"Repeating characters in title" in reasons or
"Repeating characters in body" in reasons or
"Repeating characters in answer" in reasons or
"Repeating words in title" in reasons or
"Repeating words in body" in reasons or
"Repeating words in answer" in reasons):
add_auto_ignored_post((post_id, site, datetime.now()))
if why is not None and why != "":
add_why(site, post_id, why)
try:
owner = poster_url
users_file = open("users.txt", "a")
users_file.write(site + " " + owner + " " + title + " " + post_url + "\n")
users_file.close()
except Exception as e:
print e
try:
title = escape_special_chars_in_title(title)
if not poster.strip():
s = u"[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] {}: [{}]({}) by a deleted user on `{}`" \
.format(reason, title.strip(), post_url, site)
username = ""
user_link = ""
else:
s = u"[ [SmokeDetector](https://github.com/Charcoal-SE/SmokeDetector) ] {}: [{}]({}) by [{}]({}) on `{}`" \
.format(reason, title.strip(), post_url, poster.strip(), poster_url, site)
username = poster.strip()
user_link = poster_url
t_metasmoke = Thread(target=Metasmoke.send_stats_on_post,
args=(title, post_url, reason.split(", "), body, username, user_link))
t_metasmoke.start()
print GlobalVars.parser.unescape(s).encode('ascii', errors='replace')
if time.time() >= GlobalVars.blockedTime:
append_to_latest_questions(site, post_id, title)
if reason not in GlobalVars.experimental_reasons:
GlobalVars.charcoal_hq.send_message(s)
GlobalVars.tavern_on_the_meta.send_message(s)
if site == "stackoverflow.com":
GlobalVars.socvr.send_message(s)
for specialroom in GlobalVars.specialrooms:
sites = specialroom["sites"]
if site in sites and reason not in specialroom["unwantedReasons"]:
specialroom["room"].send_message(s)
except:
print "NOP"
def handle_spam_json(data, reason, why=""):
try:
d = json.loads(json.loads(data)["data"])
title = unescape_title(d["titleEncodedFancy"])
body = d["bodySummary"]
poster = d["ownerDisplayName"]
site = d["siteBaseHostAddress"]
url = d["url"]
poster_url = d["ownerUrl"]
post_id = str(d["id"])
handle_spam(title, body, poster, site, url, poster_url, post_id, reason, False, why)
except:
print "NOP"