def unblock(msg, room_id):
"""
Unblocks posting to a room
:param msg:
:param room_id:
:return: A string
"""
block_room(room_id, msg._client.host, -1)
which_room = "globally" if room_id is None else "in room " + room_id
unblock_message = "Reports unblocked {}.".format(which_room)
tell_rooms(unblock_message, ("debug", "metatavern"), ())
return report
def handle_spam(post, reasons, why):
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.utcnow()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id != "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
username = ""
else:
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
offensive_mask = 'offensive title detected' in reasons
message = build_message(post, reasons)
if offensive_mask:
post.title = "(potentially offensive title -- see MS for details)"
clean_message = build_message(post, reasons)
log('debug', GlobalVars.parser.unescape(message).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,)
if set(reasons) - GlobalVars.experimental_reasons == set() and \
not why.startswith("Post manually "):
chatcommunicate.tell_rooms(message, ("experimental-all-sites", "experimental-site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
if offensive_mask:
chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site),
without_roles + ("offensive-mask",), notify_site=post.post_site,
report_data=(post_url, poster_url))
chatcommunicate.tell_rooms(clean_message, ("all-sites", "site-" + post.post_site),
without_roles + ("no-offensive-mask",), notify_site=post.post_site,
report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site),
without_roles, notify_site=post.post_site,
report_data=(post_url, poster_url))
except Exception as e:
excepthook.uncaught_exception(*sys.exc_info())
def block(msg, block_time, room_id):
"""
Blocks posts from application for a period of time
:param msg:
:param block_time:
:param room_id:
:return: A string
"""
time_to_block = block_time if 0 < block_time < 14400 else 900
block_room(room_id, msg._client.host, time.time() + time_to_block)
which_room = "globally" if room_id is None else "in room " + room_id
block_message = "Reports blocked for {} seconds {}.".format(time_to_block, which_room)
tell_rooms(block_message, ("debug", "metatavern"), ())
return report
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(
parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(
parsing.user_url_to_shortlink(post.user_url))
reason = ", ".join(reasons[:5])
if len(reasons) > 5:
reason += ", +{} more".format(len(reasons) - 5)
reason = reason[:1].upper() + reason[
1:] # reason is capitalised, unlike the entries of reasons list
shortened_site = post.post_site.replace(
"stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(
post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons
or "repeating characters in title" in reasons
or "repeating characters in body" in reasons
or "repeating characters in answer" in reasons
or "repeating words in title" in reasons
or "repeating words in body" in reasons
or "repeating words in answer" in reasons):
datahandling.add_auto_ignored_post(
(post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link(
(post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
post._title = parsing.escape_special_chars_in_title(post.title)
if post.is_answer:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = regex.sub('(https?://|\n)', '',
post.parent.title)
else:
sanitized_title = regex.sub('(https?://|\n)', '', post.title)
sanitized_title = regex.sub(r'([\]*`])', r'\\\1',
sanitized_title).replace('\n', u'\u23CE')
prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + \
post_url + ") ]"
else:
prefix_ms = prefix
if not post.user_name.strip() or (not poster_url
or poster_url.strip() == ""):
s = u" {}: [{}]({}) by a deleted user on `{}`".format(
reason, sanitized_title.strip(), post_url, shortened_site)
username = ""
else:
s = u" {}: [{}]({}) by [{}]({}) on `{}`".format(
reason, sanitized_title.strip(), post_url,
post.user_name.strip(), poster_url, shortened_site)
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body,
username, post.user_link, why, post.owner_rep,
post.post_score, post.up_vote_count, post.down_vote_count)
log('debug',
GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
datahandling.append_to_latest_questions(post.post_site, post.post_id,
post.title)
message = prefix_ms + s
if len(message) > 500:
message = (prefix + s)[:500]
without_roles = tuple(
"no-" + reason
for reason in reasons) + ("site-no-" + post.post_site, )
if set(reason) & GlobalVars.experimental_reasons == {}:
chatcommunicate.tell_rooms(message, ("experimental"),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message,
("all", "site-" + post.post_site),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(
parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(
parsing.user_url_to_shortlink(post.user_url))
shortened_site = post.post_site.replace(
"stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(
post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons
or "repeating characters in title" in reasons
or "repeating characters in body" in reasons
or "repeating characters in answer" in reasons
or "repeating words in title" in reasons
or "repeating words in body" in reasons
or "repeating words in answer" in reasons):
datahandling.add_auto_ignored_post(
(post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link(
(post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = parsing.sanitize_title(
post.title if not post.is_answer else post.parent.title)
prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/uid/{}/{}) ]".format(
api_parameter_from_link(post_url), post.post_id)
else:
prefix_ms = prefix
# We'll insert reason list later
if not post.user_name.strip() or (not poster_url
or poster_url.strip() == ""):
s = u" {{}}: [{}]({}) by a deleted user on `{}`".format(
sanitized_title, post_url, shortened_site)
username = ""
else:
s = u" {{}}: [{}]({}) by [{}]({}) on `{}`".format(
sanitized_title, post_url, post.user_name.strip(), poster_url,
shortened_site)
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body,
username, post.user_link, why, post.owner_rep,
post.post_score, post.up_vote_count, post.down_vote_count)
log('debug',
GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
reason = message = None
for reason_count in range(5, 2, -1): # Try 5 reasons, then 4, then 3
reason = ", ".join(reasons[:reason_count])
if len(reasons) > reason_count:
reason += ", +{} more".format(len(reasons) - reason_count)
reason = reason[:1].upper() + reason[
1:] # reason is capitalised, unlike the entries of reasons list
message = prefix_ms + s.format(reason) # Insert reason list
if len(message) <= 500:
break # Problem solved, stop attempting
s = s.format(reason) # Later code needs this variable
if len(message) > 500:
message = (prefix_ms +
s)[:500] # Truncate directly and keep MS link
without_roles = tuple(
"no-" + reason
for reason in reasons) + ("site-no-" + post.post_site, )
if set(reasons) - GlobalVars.experimental_reasons == set():
chatcommunicate.tell_rooms(message, ("experimental", ),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message,
("all", "site-" + post.post_site),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_websocket_data(data):
if "message" not in data:
if "type" in data and data['type'] == "reject_subscription":
log(
'error',
"MS WebSocket subscription was rejected. Check your MS key."
)
raise ConnectionError("MS WebSocket connection rejected")
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
message["report"]["user"], True,
"the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
recent_commits = sp.check_output(
["git", "log", "-50",
"--pretty=%H"]).decode('utf-8').strip().split('\n')
if c["commit_sha"] in recent_commits:
return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_blacklists)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart_findspam)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
elif "domain_whitelist" in message:
if message["domain_whitelist"] == "refresh":
metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return # Same rev, nothing to do
if c["status"] == "success":
if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
c["commit_message"].startswith("Auto "):
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_branch:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
exit_mode("checkout_deploy")
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
exit_mode("pull_update")
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
exit_mode("shutdown")
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(
parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(
parsing.user_url_to_shortlink(post.user_url))
shortened_site = post.post_site.replace(
"stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(
post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons
or "repeating characters in title" in reasons
or "repeating characters in body" in reasons
or "repeating characters in answer" in reasons
or "repeating words in title" in reasons
or "repeating words in body" in reasons
or "repeating words in answer" in reasons):
datahandling.add_auto_ignored_post(
(post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link(
(post.post_id, post.post_site, "answer"), post.parent.post_id)
if GlobalVars.reason_weights or GlobalVars.metasmoke_key:
reason_weight = sum_weight(reasons)
if reason_weight >= 1000:
reason_weight_s = " (**{:,}**)".format(reason_weight)
else:
reason_weight_s = " ({:,})".format(reason_weight)
else: # No reason weight if neither cache nor MS
reason_weight_s = ""
try:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = parsing.sanitize_title(
post.title if not post.is_answer else post.parent.title)
sanitized_title = escape_format(sanitized_title).strip()
prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS]({}) ]".format(
to_metasmoke_link(post_url, protocol=False))
else:
prefix_ms = prefix
# We'll insert reason list later
edited = '' if not post.edited else ' \u270F\uFE0F'
if not post.user_name.strip() or (not poster_url
or poster_url.strip() == ""):
s = " {{}}{}: [{}]({}){} by a deleted user on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited,
shortened_site)
username = ""
else:
username = post.user_name.strip()
escaped_username = escape_format(parsing.escape_markdown(username))
s = " {{}}{}: [{}]({}){} by [{}]({}) on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited,
escaped_username, poster_url, shortened_site)
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body,
username, post.user_link, why, post.owner_rep,
post.post_score, post.up_vote_count, post.down_vote_count)
log('debug',
GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
reason = message = None
for reason_count in range(
5, 0, -1): # Try 5 reasons and all the way down to 1
reason = ", ".join(reasons[:reason_count])
if len(reasons) > reason_count:
reason += ", +{} more".format(len(reasons) - reason_count)
reason = reason.capitalize()
message = prefix_ms + s.format(reason) # Insert reason list
if len(message) <= 500:
break # Problem solved, stop attempting
s = s.format(reason) # Later code needs this variable
if len(message) > 500:
message = (prefix_ms +
s)[:500] # Truncate directly and keep MS link
without_roles = tuple(["no-" + reason for reason in reasons
]) + ("site-no-" + post.post_site, )
if set(reasons) - GlobalVars.experimental_reasons == set() and \
not why.startswith("Post manually "):
chatcommunicate.tell_rooms(message, ("experimental", ),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message,
("all", "site-" + post.post_site),
without_roles,
notify_site=post.post_site,
report_data=(post_url, poster_url))
except Exception as e:
excepthook.uncaught_exception(*sys.exc_info())
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
True, "the metasmoke API")
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if not isinstance(message, Iterable):
return
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'],
message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
import chatcommands # Do it here
chatcommands.report_posts([message["report"]["post_link"]],
"the metasmoke API", None,
"the metasmoke API")
return
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
post = classes.Post(api_response=post_data.as_dict)
scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(
post)
if scan_spam:
why_append = u"This post would have also been caught for: " + \
u", ".join(scan_reasons).capitalize() + "\n" + scan_why
else:
why_append = u"This post would not have been caught otherwise."
# Add user to blacklist *after* post is scanned
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
message["report"]["user"], why_append)
spamhandling.handle_spam(
post=post,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
return # Disabled
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
.format(sha[:7], GlobalVars.bot_repo_slug)
integrity = blacklist_integrity_check()
if len(integrity) == 0: # No issues
GitManager.pull_remote()
findspam.reload_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
integrity.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(integrity))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] == sp.check_output(
["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
return
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
"commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
remote_diff = GitManager.get_remote_diff()
if only_blacklists_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
findspam.FindSpam.reload_blacklists()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart)
elif only_modules_changed(remote_diff):
GitManager.pull_remote()
if not GlobalVars.on_master:
# Restart if HEAD detached
log('warning',
"Pulling remote with HEAD detached, checkout deploy",
f=True)
os._exit(8)
GlobalVars.reload()
reload_modules()
chatcommunicate.tell_rooms_with(
'debug', GlobalVars.s_norestart2)
else:
chatcommunicate.tell_rooms_with('debug',
s,
notify_site="/ci")
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
"failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke",
message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"],
("debug", "site-" + event["site"]),
("no-site-" + event["site"], ),
notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'],
message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke",
message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
datahandling.remove_blacklisted_user(
message['unblacklist']['uid'])
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(
message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n".format(
message["report"]["user"])
postobj = classes.Post(
api_response={
'title': post_data.title,
'body': post_data.body,
'owner': {
'display_name': post_data.owner_name,
'reputation': post_data.owner_rep,
'link': post_data.owner_url
},
'site': post_data.site,
'is_answer': (post_data.post_type == "answer"),
'score': post_data.score,
'link': post_data.post_url,
'question_id': post_data.post_id,
'up_vote_count': post_data.up_vote_count,
'down_vote_count': post_data.down_vote_count
})
spamhandling.handle_spam(
post=postobj,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(
GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob(
'blacklisted_*.txt'
): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append(
"DOS line ending at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append(
"No newline at end of `{0}` in {1}"
.format(bl_file, commit_md))
if line == '\n':
i.append(
"Blank line at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if line in seen:
i.append(
"Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(
line.rstrip('\n'),
seen[line], lineno,
bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
chatcommunicate.tell_rooms_with(
"debug",
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
chatcommunicate.tell_rooms_with(
"debug", ", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen(
'git log --pretty=format:"%H" -n 1').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug",
s,
notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
reason = ", ".join(reasons[:5])
if len(reasons) > 5:
reason += ", +{} more".format(len(reasons) - 5)
reason = reason[:1].upper() + reason[1:] # reason is capitalised, unlike the entries of reasons list
shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
try:
post._title = parsing.escape_special_chars_in_title(post.title)
if post.is_answer:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = regex.sub('(https?://|\n)', '', post.parent.title)
else:
sanitized_title = regex.sub('(https?://|\n)', '', post.title)
sanitized_title = regex.sub(r'([\]*`])', r'\\\1', sanitized_title).replace('\n', u'\u23CE')
prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + \
post_url + ") ]"
else:
prefix_ms = prefix
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
s = u" {}: [{}]({}) by a deleted user on `{}`".format(reason, sanitized_title.strip(), post_url,
shortened_site)
username = ""
else:
s = u" {}: [{}]({}) by [{}]({}) on `{}`".format(reason, sanitized_title.strip(), post_url,
post.user_name.strip(), poster_url, shortened_site)
username = post.user_name.strip()
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title)
message = prefix_ms + s
if len(message) > 500:
message = (prefix + s)[:500]
without_roles = tuple("no-" + reason for reason in reasons)
if set(reason) & GlobalVars.experimental_reasons == {}:
chatcommunicate.tell_rooms(message, ("experimental"),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
except:
exc_type, exc_obj, exc_tb = sys.exc_info()
excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_spam(post, reasons, why):
post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url))
poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url))
shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE
datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "")
if len(reasons) == 1 and ("all-caps title" in reasons or
"repeating characters in title" in reasons or
"repeating characters in body" in reasons or
"repeating characters in answer" in reasons or
"repeating words in title" in reasons or
"repeating words in body" in reasons or
"repeating words in answer" in reasons):
datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now()))
if why is not None and why != "":
datahandling.add_why(post.post_site, post.post_id, why)
if post.is_answer and post.post_id is not None and post.post_id is not "":
datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id)
if GlobalVars.reason_weights or GlobalVars.metasmoke_key:
reason_weight = sum_weight(reasons)
if reason_weight >= 1000:
reason_weight_s = " (**{}**)".format(reason_weight)
else:
reason_weight_s = " ({})".format(reason_weight)
else: # No reason weight if neither cache nor MS
reason_weight_s = ""
try:
# If the post is an answer type post, the 'title' is going to be blank, so when posting the
# message contents we need to set the post title to the *parent* title, so the message in the
# chat is properly constructed with parent title instead. This will make things 'print'
# in a proper way in chat messages.
sanitized_title = parsing.sanitize_title(post.title if not post.is_answer else post.parent.title)
sanitized_title = escape_format(sanitized_title).strip()
prefix = u"[ [SmokeDetector](//git.io/vyDZv) ]"
if GlobalVars.metasmoke_key:
prefix_ms = u"[ [SmokeDetector](//git.io/vyDZv) | [MS]({}) ]".format(
to_metasmoke_link(post_url, protocol=False))
else:
prefix_ms = prefix
# We'll insert reason list later
edited = '' if not post.edited else ' \u270F\uFE0F'
if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""):
s = " {{}}{}: [{}]({}){} by a deleted user on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited, shortened_site)
username = ""
else:
username = post.user_name.strip()
escaped_username = escape_format(parsing.escape_markdown(username))
s = " {{}}{}: [{}]({}){} by [{}]({}) on `{}`".format(
reason_weight_s, sanitized_title, post_url, edited, escaped_username, poster_url, shortened_site)
Tasks.do(metasmoke.Metasmoke.send_stats_on_post,
post.title_ignore_type, post_url, reasons, post.body, username,
post.user_link, why, post.owner_rep, post.post_score,
post.up_vote_count, post.down_vote_count)
log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace'))
GlobalVars.deletion_watcher.subscribe(post_url)
reason = message = None
for reason_count in range(5, 0, -1): # Try 5 reasons and all the way down to 1
reason = ", ".join(reasons[:reason_count])
if len(reasons) > reason_count:
reason += ", +{} more".format(len(reasons) - reason_count)
reason = reason.capitalize()
message = prefix_ms + s.format(reason) # Insert reason list
if len(message) <= 500:
break # Problem solved, stop attempting
s = s.format(reason) # Later code needs this variable
if len(message) > 500:
message = (prefix_ms + s)[:500] # Truncate directly and keep MS link
without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,)
if set(reasons) - GlobalVars.experimental_reasons == set() and \
not why.startswith("Post manually "):
chatcommunicate.tell_rooms(message, ("experimental",),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
else:
chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site),
without_roles, notify_site=post.post_site, report_data=(post_url, poster_url))
except Exception as e:
excepthook.uncaught_exception(*sys.exc_info())
