def load_files():
if os.path.isfile("falsePositives.p"):
GlobalVars.false_positives = _load_pickle("falsePositives.p",
encoding='utf-8')
if os.path.isfile("whitelistedUsers.p"):
GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p",
encoding='utf-8')
if os.path.isfile("blacklistedUsers.p"):
GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p",
encoding='utf-8')
if os.path.isfile("ignoredPosts.p"):
GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p",
encoding='utf-8')
if os.path.isfile("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p",
encoding='utf-8')
if os.path.isfile("notifications.p"):
GlobalVars.notifications = _load_pickle("notifications.p",
encoding='utf-8')
if os.path.isfile("whyData.p"):
GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8')
if os.path.isfile("apiCalls.p"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p",
encoding='utf-8')
if os.path.isfile("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p",
encoding='utf-8')
if os.path.isfile("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = _load_pickle(
"bodyfetcherMaxIds.p", encoding='utf-8')
if os.path.isfile("bodyfetcherQueueTimings.p"):
GlobalVars.bodyfetcher.queue_timings = _load_pickle(
"bodyfetcherQueueTimings.p", encoding='utf-8')
load_blacklists()
def pull():
"""
Pull an update from GitHub
:return: String on failure, None on success
"""
if only_blacklists_changed(GitManager.get_remote_diff()):
GitManager.pull_remote()
load_blacklists()
return "No code modified, only blacklists reloaded."
else:
request = requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/git/refs/heads/deploy')
latest_sha = request.json()["object"]["sha"]
request = requests.get(
'https://api.github.com/repos/Charcoal-SE/SmokeDetector/commits/{commit_code}/statuses'.format(
commit_code=latest_sha))
states = []
for ci_status in request.json():
state = ci_status["state"]
states.append(state)
if "success" in states:
os._exit(3)
elif "error" in states or "failure" in states:
raise CmdException("CI build failed! :( Please check your commit.")
elif "pending" in states or not states:
raise CmdException("CI build is still pending, wait until the build has finished and then pull again.")
def load_files():
if _has_pickle("falsePositives.p"):
GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8')
if _has_pickle("whitelistedUsers.p"):
GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8')
if _has_pickle("blacklistedUsers.p"):
GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8')
if _has_pickle("ignoredPosts.p"):
GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8')
if _has_pickle("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8')
if _has_pickle("notifications.p"):
GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8')
if _has_pickle("whyData.p"):
GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8')
if _has_pickle("apiCalls.p"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8')
if _has_pickle("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueueTimings.p"):
GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8')
if _has_pickle("codePrivileges.p"):
GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8')
if _has_pickle("reasonWeights.p"):
GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8')
blacklists.load_blacklists()
def load_files():
if _has_pickle("falsePositives.p"):
GlobalVars.false_positives = _load_pickle("falsePositives.p",
encoding='utf-8')
if _has_pickle("whitelistedUsers.p"):
GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p",
encoding='utf-8')
if not isinstance(GlobalVars.whitelisted_users, set):
GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users)
if _has_pickle("blacklistedUsers.p"):
GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p",
encoding='utf-8')
if not isinstance(GlobalVars.blacklisted_users, dict):
GlobalVars.blacklisted_users = {
data[0]: data[1:]
for data in GlobalVars.blacklisted_users
}
if _has_pickle("ignoredPosts.p"):
GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p",
encoding='utf-8')
if _has_pickle("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p",
encoding='utf-8')
if _has_pickle("notifications.p"):
GlobalVars.notifications = _load_pickle("notifications.p",
encoding='utf-8')
if _has_pickle("whyData.p"):
GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8')
if _has_pickle("apiCalls.p"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p",
encoding='utf-8')
if _has_pickle("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p",
encoding='utf-8')
if _has_pickle("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = _load_pickle(
"bodyfetcherMaxIds.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueueTimings.p"):
GlobalVars.bodyfetcher.queue_timings = _load_pickle(
"bodyfetcherQueueTimings.p", encoding='utf-8')
if _has_pickle("codePrivileges.p"):
GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p",
encoding='utf-8')
if _has_pickle("reasonWeights.p"):
GlobalVars.reason_weights = _load_pickle("reasonWeights.p",
encoding='utf-8')
if _has_pickle("cookies.p"):
GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8')
if _has_pickle("metasmokePostIds.p"):
GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p",
encoding='utf-8')
blacklists.load_blacklists()
def load_files():
if _has_pickle("falsePositives.p"):
GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8')
if _has_pickle("whitelistedUsers.p"):
GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8')
if not isinstance(GlobalVars.whitelisted_users, set):
GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users)
if _has_pickle("blacklistedUsers.p"):
GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8')
if not isinstance(GlobalVars.blacklisted_users, dict):
GlobalVars.blacklisted_users = {data[0]: data[1:] for data in GlobalVars.blacklisted_users}
if _has_pickle("ignoredPosts.p"):
GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8')
if _has_pickle("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8')
if _has_pickle("notifications.p"):
GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8')
if _has_pickle("whyData.p"):
GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8')
# Switch from apiCalls.pickle to apiCalls.p
# Correction was on 2020-11-02. Handling the apiCalls.pickle file should be able to be removed shortly thereafter.
if _has_pickle("apiCalls.pickle"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.pickle", encoding='utf-8')
# Remove the incorrectly named pickle file.
_remove_pickle("apiCalls.pickle")
# Put the pickle in the "correct" file, from which it will be immediately reloaded.
_dump_pickle("apiCalls.p", GlobalVars.api_calls_per_site)
if _has_pickle("apiCalls.p"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8')
if _has_pickle("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueueTimings.p"):
GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8')
if _has_pickle("codePrivileges.p"):
GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8')
if _has_pickle("reasonWeights.p"):
GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8')
if _has_pickle("cookies.p"):
GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8')
if _has_pickle("metasmokePostIds.p"):
GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p", encoding='utf-8')
blacklists.load_blacklists()
def load_files():
if _has_pickle("falsePositives.p"):
GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8')
if _has_pickle("whitelistedUsers.p"):
GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8')
if not isinstance(GlobalVars.whitelisted_users, set):
GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users)
if _has_pickle("blacklistedUsers.p"):
GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8')
if not isinstance(GlobalVars.blacklisted_users, dict):
GlobalVars.blacklisted_users = {data[0]: data[1:] for data in GlobalVars.blacklisted_users}
if _has_pickle("ignoredPosts.p"):
GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8')
if _has_pickle("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8')
if _has_pickle("notifications.p"):
GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8')
if _has_pickle("whyData.p"):
GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8')
if _has_pickle("apiCalls.p"):
GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8')
if _has_pickle("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8')
if _has_pickle("bodyfetcherQueueTimings.p"):
GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8')
if _has_pickle("codePrivileges.p"):
GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8')
if _has_pickle("reasonWeights.p"):
GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8')
if _has_pickle("cookies.p"):
GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8')
if _has_pickle("metasmokePostIds.p"):
GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p", encoding='utf-8')
blacklists.load_blacklists()
# coding=utf-8
from spamhandling import check_if_spam, check_if_spam_json, handle_spam
from datahandling import add_blacklisted_user, add_whitelisted_user, _remove_pickle
from blacklists import load_blacklists
from parsing import get_user_from_url
import pytest
import os
import json
from classes import Post
from unittest import TestCase
from unittest.mock import MagicMock, call, ANY, DEFAULT
import chatcommunicate
from globalvars import GlobalVars
load_blacklists()
test_data_inputs = []
with open("test/data_test_spamhandling.txt", "r", encoding="utf-8") as f:
# noinspection PyRedeclaration
test_data_inputs = f.readlines()
class Matcher:
def __init__(self, containing, without):
self.containing = containing
self.without = without
def __eq__(self, other):
return self.containing in other and self.without not in other
# coding=utf-8
from spamhandling import check_if_spam, check_if_spam_json
from datahandling import add_blacklisted_user, add_whitelisted_user
from blacklists import load_blacklists
from parsing import get_user_from_url
import pytest
import os
import json
from classes import Post
load_blacklists()
test_data_inputs = []
with open("test/data_test_spamhandling.txt", "r", encoding="utf-8") as f:
# noinspection PyRedeclaration
test_data_inputs = f.readlines()
# noinspection PyMissingTypeHints
@pytest.mark.parametrize("title, body, username, site, match", [
('18669786819 gmail customer service number 1866978-6819 gmail support number', '', '', '', True),
('Is there any http://www.hindawi.com/ template for Cloud-Oriented Data Center Networking?', '', '', '', True),
('', '', 'bagprada', '', True),
('12 Month Loans quick @ http://www.quick12monthpaydayloans.co.uk/Elimination of collateral pledging', '', '', '', True),
('support for yahoo mail 18669786819 @call for helpline number', '', '', '', True),
('yahoo email tech support 1 866 978 6819 Yahoo Customer Phone Number ,Shortest Wait', '', '', '', True),
('Not a phone number 192.168.0.1', 'Not a phone number 192.168.0.1', '', '', False),
('What is the value of MD5 checksums if the MD5 hash itself could potentially also have been manipulated?', '', '', '', False),
('Probability: 6 Dice are rolled. Which is more likely, that you get exactly one 6, or that you get 6 different numbers?', '', '', '', False),
('The Challenge of Controlling a Powerful AI', '', 'Serban Tanasa', '', False),
('Reproducing image of a spiral using TikZ', '', 'Kristoffer Ryhl', '', False),
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
GlobalVars.charcoal_hq.send_message(message['message'])
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
datahandling.add_blacklisted_user(
(message['blacklist']['uid'],
message['blacklist']['site']), "metasmoke",
message['blacklist']['post'])
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(
message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(
message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke",
post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n".format(
message["report"]["user"])
postobj = classes.Post(
api_response={
'title': post_data.title,
'body': post_data.body,
'owner': {
'display_name': post_data.owner_name,
'reputation': post_data.owner_rep,
'link': post_data.owner_url
},
'site': post_data.site,
'IsAnswer': (post_data.post_type == "answer"),
'score': post_data.score,
'link': post_data.post_url,
'question_id': post_data.post_id,
'up_vote_count': post_data.up_vote_count,
'down_vote_count': post_data.down_vote_count
})
spamhandling.handle_spam(
post=postobj,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
if "autopull" in message["deploy_updated"]["head_commit"][
"message"]:
if only_blacklists_changed(
GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob(
'blacklisted_*.txt'
): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append(
"DOS line ending at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append(
"No newline at end of `{0}` in {1}"
.format(bl_file, commit_md))
if line == '\n':
i.append(
"Blank line at `{0}:{1}` in {2}"
.format(
bl_file, lineno,
commit_md))
if line in seen:
i.append(
"Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(
line.rstrip('\n'),
seen[line], lineno,
bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
GlobalVars.charcoal_hq.send_message(
"No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
GlobalVars.charcoal_hq.send_message(
", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen(
'git log --pretty=format:"%H" -n 1').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
GlobalVars.charcoal_hq.send_message(s)
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
# noinspection PyUnboundLocalVariable
GlobalVars.charcoal_hq.send_message(s)
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def load_files():
if has_pickle("falsePositives.p"):
GlobalVars.false_positives = load_pickle("falsePositives.p",
encoding='utf-8')
if has_pickle("whitelistedUsers.p"):
GlobalVars.whitelisted_users = load_pickle("whitelistedUsers.p",
encoding='utf-8')
if not isinstance(GlobalVars.whitelisted_users, set):
GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users)
if has_pickle("blacklistedUsers.p"):
GlobalVars.blacklisted_users = load_pickle("blacklistedUsers.p",
encoding='utf-8')
if not isinstance(GlobalVars.blacklisted_users, dict):
GlobalVars.blacklisted_users = {
data[0]: data[1:]
for data in GlobalVars.blacklisted_users
}
if has_pickle("ignoredPosts.p"):
GlobalVars.ignored_posts = load_pickle("ignoredPosts.p",
encoding='utf-8')
if has_pickle("autoIgnoredPosts.p"):
GlobalVars.auto_ignored_posts = load_pickle("autoIgnoredPosts.p",
encoding='utf-8')
if has_pickle("notifications.p"):
GlobalVars.notifications = load_pickle("notifications.p",
encoding='utf-8')
if has_pickle("whyData.p"):
GlobalVars.why_data = load_pickle("whyData.p", encoding='utf-8')
# Switch from apiCalls.pickle to apiCalls.p
# Correction was on 2020-11-02. Handling the apiCalls.pickle file should be able to be removed shortly thereafter.
if has_pickle("apiCalls.pickle"):
GlobalVars.api_calls_per_site = load_pickle("apiCalls.pickle",
encoding='utf-8')
# Remove the incorrectly named pickle file.
remove_pickle("apiCalls.pickle")
# Put the pickle in the "correct" file, from which it will be immediately reloaded.
dump_pickle("apiCalls.p", GlobalVars.api_calls_per_site)
if has_pickle("apiCalls.p"):
GlobalVars.api_calls_per_site = load_pickle("apiCalls.p",
encoding='utf-8')
if has_pickle("bodyfetcherQueue.p"):
GlobalVars.bodyfetcher.queue = load_pickle("bodyfetcherQueue.p",
encoding='utf-8')
if has_pickle("bodyfetcherMaxIds.p"):
GlobalVars.bodyfetcher.previous_max_ids = load_pickle(
"bodyfetcherMaxIds.p", encoding='utf-8')
if has_pickle("codePrivileges.p"):
GlobalVars.code_privileged_users = load_pickle("codePrivileges.p",
encoding='utf-8')
if has_pickle("reasonWeights.p"):
GlobalVars.reason_weights = load_pickle("reasonWeights.p",
encoding='utf-8')
if has_pickle("cookies.p"):
GlobalVars.cookies = load_pickle("cookies.p", encoding='utf-8')
if has_pickle("metasmokePostIds.p"):
GlobalVars.metasmoke_ids = load_pickle("metasmokePostIds.p",
encoding='utf-8')
if has_pickle("ms_ajax_queue.p"):
with metasmoke.Metasmoke.ms_ajax_queue_lock:
metasmoke.Metasmoke.ms_ajax_queue = load_pickle("ms_ajax_queue.p")
log(
"debug", "Loaded {} entries into ms_ajax_queue".format(
len(metasmoke.Metasmoke.ms_ajax_queue)))
if has_pickle("seSiteIds.p"):
with GlobalVars.site_id_dict_lock:
(GlobalVars.site_id_dict_timestamp,
GlobalVars.site_id_dict_issues_into_chat_timestamp,
GlobalVars.site_id_dict) = load_pickle("seSiteIds.p",
encoding='utf-8')
fill_site_id_dict_by_id_from_site_id_dict()
if has_pickle("recentlyScannedPosts.p"):
with GlobalVars.recently_scanned_posts_lock:
GlobalVars.recently_scanned_posts = load_pickle(
"recentlyScannedPosts.p", encoding='utf-8')
blacklists.load_blacklists()
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
chatcommunicate.tell_rooms_with("debug", message['message'])
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
"metasmoke", message['blacklist']['post'])
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
'owner': {'display_name': post_data.owner_name,
'reputation': post_data.owner_rep,
'link': post_data.owner_url},
'site': post_data.site,
'is_answer': (post_data.post_type == "answer"),
'score': post_data.score, 'link': post_data.post_url,
'question_id': post_data.post_id,
'up_vote_count': post_data.up_vote_count,
'down_vote_count': post_data.down_vote_count})
spamhandling.handle_spam(post=postobj,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
if line == '\n':
i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if line in seen:
i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s)
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
def handle_websocket_data(data):
if "message" not in data:
return
message = data['message']
if isinstance(message, Iterable):
if "message" in message:
chatcommunicate.tell_rooms_with("metasmoke", message['message'])
elif "autoflag_fp" in message:
event = message["autoflag_fp"]
chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
("no-site-" + event["site"],), notify_site="/autoflag_fp")
elif "exit" in message:
os._exit(message["exit"])
elif "blacklist" in message:
ids = (message['blacklist']['uid'], message['blacklist']['site'])
datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
datahandling.last_feedbacked = (ids, time.time() + 60)
elif "unblacklist" in message:
ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
datahandling.remove_blacklisted_user(ids)
elif "naa" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
datahandling.add_ignored_post(post_site_id[0:2])
elif "fp" in message:
post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
datahandling.add_false_positive(post_site_id[0:2])
elif "report" in message:
post_data = apigetpost.api_get_post(message["report"]["post_link"])
if post_data is None or post_data is False:
return
if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
return
user = parsing.get_user_from_url(post_data.owner_url)
post = classes.Post(api_response=post_data.as_dict)
scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(post)
if scan_spam:
why_append = u"This post would have also been caught for: " + \
u", ".join(scan_reasons).capitalize() + "\n" + scan_why
else:
why_append = u"This post would not have been caught otherwise."
# Add user to blacklist *after* post is scanned
if user is not None:
datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
message["report"]["user"], why_append)
spamhandling.handle_spam(post=post,
reasons=["Manually reported " + post_data.post_type],
why=why)
elif "deploy_updated" in message:
sha = message["deploy_updated"]["head_commit"]["id"]
if sha != os.popen('git log -1 --pretty="%H"').read():
if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
if only_blacklists_changed(GitManager.get_remote_diff()):
commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
.format(sha[:7])
i = [] # Currently no issues with backlists
for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues
with open(bl_file, 'r') as lines:
seen = dict()
for lineno, line in enumerate(lines, 1):
if line.endswith('\r\n'):
i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if not line.endswith('\n'):
i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
if line == '\n':
i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
commit_md))
if line in seen:
i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
.format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
seen[line] = lineno
if i == []: # No issues
GitManager.pull_remote()
load_blacklists()
chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
" reloaded.".format(commit_md))
else:
i.append("please fix before pulling.")
chatcommunicate.tell_rooms_with("debug", ", ".join(i))
elif "commit_status" in message:
c = message["commit_status"]
sha = c["commit_sha"][:7]
if c["commit_sha"] != os.popen('git log -1 --pretty="%H"').read():
if c["status"] == "success":
if "autopull" in c["commit_message"]:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha})"\
" succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
time.sleep(2)
os._exit(3)
else:
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif c["status"] == "failure":
s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
"commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)
chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
elif "everything_is_broken" in message:
if message["everything_is_broken"] is True:
os._exit(6)
