def load_files(): if os.path.isfile("falsePositives.p"): GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8') if os.path.isfile("whitelistedUsers.p"): GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8') if os.path.isfile("blacklistedUsers.p"): GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8') if os.path.isfile("ignoredPosts.p"): GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8') if os.path.isfile("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8') if os.path.isfile("notifications.p"): GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8') if os.path.isfile("whyData.p"): GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8') if os.path.isfile("apiCalls.p"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8') if os.path.isfile("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8') if os.path.isfile("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = _load_pickle( "bodyfetcherMaxIds.p", encoding='utf-8') if os.path.isfile("bodyfetcherQueueTimings.p"): GlobalVars.bodyfetcher.queue_timings = _load_pickle( "bodyfetcherQueueTimings.p", encoding='utf-8') load_blacklists()
def pull(): """ Pull an update from GitHub :return: String on failure, None on success """ if only_blacklists_changed(GitManager.get_remote_diff()): GitManager.pull_remote() load_blacklists() return "No code modified, only blacklists reloaded." else: request = requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/git/refs/heads/deploy') latest_sha = request.json()["object"]["sha"] request = requests.get( 'https://api.github.com/repos/Charcoal-SE/SmokeDetector/commits/{commit_code}/statuses'.format( commit_code=latest_sha)) states = [] for ci_status in request.json(): state = ci_status["state"] states.append(state) if "success" in states: os._exit(3) elif "error" in states or "failure" in states: raise CmdException("CI build failed! :( Please check your commit.") elif "pending" in states or not states: raise CmdException("CI build is still pending, wait until the build has finished and then pull again.")
def load_files(): if _has_pickle("falsePositives.p"): GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8') if _has_pickle("whitelistedUsers.p"): GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8') if _has_pickle("blacklistedUsers.p"): GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8') if _has_pickle("ignoredPosts.p"): GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8') if _has_pickle("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8') if _has_pickle("notifications.p"): GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8') if _has_pickle("whyData.p"): GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8') if _has_pickle("apiCalls.p"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8') if _has_pickle("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8') if _has_pickle("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8') if _has_pickle("bodyfetcherQueueTimings.p"): GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8') if _has_pickle("codePrivileges.p"): GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8') if _has_pickle("reasonWeights.p"): GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8') blacklists.load_blacklists()
def load_files(): if _has_pickle("falsePositives.p"): GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8') if _has_pickle("whitelistedUsers.p"): GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.whitelisted_users, set): GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users) if _has_pickle("blacklistedUsers.p"): GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.blacklisted_users, dict): GlobalVars.blacklisted_users = { data[0]: data[1:] for data in GlobalVars.blacklisted_users } if _has_pickle("ignoredPosts.p"): GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8') if _has_pickle("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8') if _has_pickle("notifications.p"): GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8') if _has_pickle("whyData.p"): GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8') if _has_pickle("apiCalls.p"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8') if _has_pickle("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8') if _has_pickle("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = _load_pickle( "bodyfetcherMaxIds.p", encoding='utf-8') if _has_pickle("bodyfetcherQueueTimings.p"): GlobalVars.bodyfetcher.queue_timings = _load_pickle( "bodyfetcherQueueTimings.p", encoding='utf-8') if _has_pickle("codePrivileges.p"): GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8') if _has_pickle("reasonWeights.p"): GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8') if _has_pickle("cookies.p"): GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8') if _has_pickle("metasmokePostIds.p"): GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p", encoding='utf-8') blacklists.load_blacklists()
def load_files(): if _has_pickle("falsePositives.p"): GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8') if _has_pickle("whitelistedUsers.p"): GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.whitelisted_users, set): GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users) if _has_pickle("blacklistedUsers.p"): GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.blacklisted_users, dict): GlobalVars.blacklisted_users = {data[0]: data[1:] for data in GlobalVars.blacklisted_users} if _has_pickle("ignoredPosts.p"): GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8') if _has_pickle("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8') if _has_pickle("notifications.p"): GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8') if _has_pickle("whyData.p"): GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8') # Switch from apiCalls.pickle to apiCalls.p # Correction was on 2020-11-02. Handling the apiCalls.pickle file should be able to be removed shortly thereafter. if _has_pickle("apiCalls.pickle"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.pickle", encoding='utf-8') # Remove the incorrectly named pickle file. _remove_pickle("apiCalls.pickle") # Put the pickle in the "correct" file, from which it will be immediately reloaded. _dump_pickle("apiCalls.p", GlobalVars.api_calls_per_site) if _has_pickle("apiCalls.p"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8') if _has_pickle("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8') if _has_pickle("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8') if _has_pickle("bodyfetcherQueueTimings.p"): GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8') if _has_pickle("codePrivileges.p"): GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8') if _has_pickle("reasonWeights.p"): GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8') if _has_pickle("cookies.p"): GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8') if _has_pickle("metasmokePostIds.p"): GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p", encoding='utf-8') blacklists.load_blacklists()
def load_files(): if _has_pickle("falsePositives.p"): GlobalVars.false_positives = _load_pickle("falsePositives.p", encoding='utf-8') if _has_pickle("whitelistedUsers.p"): GlobalVars.whitelisted_users = _load_pickle("whitelistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.whitelisted_users, set): GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users) if _has_pickle("blacklistedUsers.p"): GlobalVars.blacklisted_users = _load_pickle("blacklistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.blacklisted_users, dict): GlobalVars.blacklisted_users = {data[0]: data[1:] for data in GlobalVars.blacklisted_users} if _has_pickle("ignoredPosts.p"): GlobalVars.ignored_posts = _load_pickle("ignoredPosts.p", encoding='utf-8') if _has_pickle("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = _load_pickle("autoIgnoredPosts.p", encoding='utf-8') if _has_pickle("notifications.p"): GlobalVars.notifications = _load_pickle("notifications.p", encoding='utf-8') if _has_pickle("whyData.p"): GlobalVars.why_data = _load_pickle("whyData.p", encoding='utf-8') if _has_pickle("apiCalls.p"): GlobalVars.api_calls_per_site = _load_pickle("apiCalls.p", encoding='utf-8') if _has_pickle("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = _load_pickle("bodyfetcherQueue.p", encoding='utf-8') if _has_pickle("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = _load_pickle("bodyfetcherMaxIds.p", encoding='utf-8') if _has_pickle("bodyfetcherQueueTimings.p"): GlobalVars.bodyfetcher.queue_timings = _load_pickle("bodyfetcherQueueTimings.p", encoding='utf-8') if _has_pickle("codePrivileges.p"): GlobalVars.code_privileged_users = _load_pickle("codePrivileges.p", encoding='utf-8') if _has_pickle("reasonWeights.p"): GlobalVars.reason_weights = _load_pickle("reasonWeights.p", encoding='utf-8') if _has_pickle("cookies.p"): GlobalVars.cookies = _load_pickle("cookies.p", encoding='utf-8') if _has_pickle("metasmokePostIds.p"): GlobalVars.metasmoke_ids = _load_pickle("metasmokePostIds.p", encoding='utf-8') blacklists.load_blacklists()
# coding=utf-8 from spamhandling import check_if_spam, check_if_spam_json, handle_spam from datahandling import add_blacklisted_user, add_whitelisted_user, _remove_pickle from blacklists import load_blacklists from parsing import get_user_from_url import pytest import os import json from classes import Post from unittest import TestCase from unittest.mock import MagicMock, call, ANY, DEFAULT import chatcommunicate from globalvars import GlobalVars load_blacklists() test_data_inputs = [] with open("test/data_test_spamhandling.txt", "r", encoding="utf-8") as f: # noinspection PyRedeclaration test_data_inputs = f.readlines() class Matcher: def __init__(self, containing, without): self.containing = containing self.without = without def __eq__(self, other): return self.containing in other and self.without not in other
# coding=utf-8 from spamhandling import check_if_spam, check_if_spam_json from datahandling import add_blacklisted_user, add_whitelisted_user from blacklists import load_blacklists from parsing import get_user_from_url import pytest import os import json from classes import Post load_blacklists() test_data_inputs = [] with open("test/data_test_spamhandling.txt", "r", encoding="utf-8") as f: # noinspection PyRedeclaration test_data_inputs = f.readlines() # noinspection PyMissingTypeHints @pytest.mark.parametrize("title, body, username, site, match", [ ('18669786819 gmail customer service number 1866978-6819 gmail support number', '', '', '', True), ('Is there any http://www.hindawi.com/ template for Cloud-Oriented Data Center Networking?', '', '', '', True), ('', '', 'bagprada', '', True), ('12 Month Loans quick @ http://www.quick12monthpaydayloans.co.uk/Elimination of collateral pledging', '', '', '', True), ('support for yahoo mail 18669786819 @call for helpline number', '', '', '', True), ('yahoo email tech support 1 866 978 6819 Yahoo Customer Phone Number ,Shortest Wait', '', '', '', True), ('Not a phone number 192.168.0.1', 'Not a phone number 192.168.0.1', '', '', False), ('What is the value of MD5 checksums if the MD5 hash itself could potentially also have been manipulated?', '', '', '', False), ('Probability: 6 Dice are rolled. Which is more likely, that you get exactly one 6, or that you get 6 different numbers?', '', '', '', False), ('The Challenge of Controlling a Powerful AI', '', 'Serban Tanasa', '', False), ('Reproducing image of a spiral using TikZ', '', 'Kristoffer Ryhl', '', False),
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message['message']) elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: datahandling.add_blacklisted_user( (message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post( message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format( message["report"]["user"]) postobj = classes.Post( api_response={ 'title': post_data.title, 'body': post_data.body, 'owner': { 'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url }, 'site': post_data.site, 'IsAnswer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count }) spamhandling.handle_spam( post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log --pretty=format:"%H" -n 1').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed( GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob( 'blacklisted_*.txt' ): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append( "DOS line ending at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append( "No newline at end of `{0}` in {1}" .format(bl_file, commit_md)) if line == '\n': i.append( "Blank line at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if line in seen: i.append( "Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format( line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() GlobalVars.charcoal_hq.send_message( "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") GlobalVars.charcoal_hq.send_message( ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen( 'git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) GlobalVars.charcoal_hq.send_message(s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) # noinspection PyUnboundLocalVariable GlobalVars.charcoal_hq.send_message(s) elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def load_files(): if has_pickle("falsePositives.p"): GlobalVars.false_positives = load_pickle("falsePositives.p", encoding='utf-8') if has_pickle("whitelistedUsers.p"): GlobalVars.whitelisted_users = load_pickle("whitelistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.whitelisted_users, set): GlobalVars.whitelisted_users = set(GlobalVars.whitelisted_users) if has_pickle("blacklistedUsers.p"): GlobalVars.blacklisted_users = load_pickle("blacklistedUsers.p", encoding='utf-8') if not isinstance(GlobalVars.blacklisted_users, dict): GlobalVars.blacklisted_users = { data[0]: data[1:] for data in GlobalVars.blacklisted_users } if has_pickle("ignoredPosts.p"): GlobalVars.ignored_posts = load_pickle("ignoredPosts.p", encoding='utf-8') if has_pickle("autoIgnoredPosts.p"): GlobalVars.auto_ignored_posts = load_pickle("autoIgnoredPosts.p", encoding='utf-8') if has_pickle("notifications.p"): GlobalVars.notifications = load_pickle("notifications.p", encoding='utf-8') if has_pickle("whyData.p"): GlobalVars.why_data = load_pickle("whyData.p", encoding='utf-8') # Switch from apiCalls.pickle to apiCalls.p # Correction was on 2020-11-02. Handling the apiCalls.pickle file should be able to be removed shortly thereafter. if has_pickle("apiCalls.pickle"): GlobalVars.api_calls_per_site = load_pickle("apiCalls.pickle", encoding='utf-8') # Remove the incorrectly named pickle file. remove_pickle("apiCalls.pickle") # Put the pickle in the "correct" file, from which it will be immediately reloaded. dump_pickle("apiCalls.p", GlobalVars.api_calls_per_site) if has_pickle("apiCalls.p"): GlobalVars.api_calls_per_site = load_pickle("apiCalls.p", encoding='utf-8') if has_pickle("bodyfetcherQueue.p"): GlobalVars.bodyfetcher.queue = load_pickle("bodyfetcherQueue.p", encoding='utf-8') if has_pickle("bodyfetcherMaxIds.p"): GlobalVars.bodyfetcher.previous_max_ids = load_pickle( "bodyfetcherMaxIds.p", encoding='utf-8') if has_pickle("codePrivileges.p"): GlobalVars.code_privileged_users = load_pickle("codePrivileges.p", encoding='utf-8') if has_pickle("reasonWeights.p"): GlobalVars.reason_weights = load_pickle("reasonWeights.p", encoding='utf-8') if has_pickle("cookies.p"): GlobalVars.cookies = load_pickle("cookies.p", encoding='utf-8') if has_pickle("metasmokePostIds.p"): GlobalVars.metasmoke_ids = load_pickle("metasmokePostIds.p", encoding='utf-8') if has_pickle("ms_ajax_queue.p"): with metasmoke.Metasmoke.ms_ajax_queue_lock: metasmoke.Metasmoke.ms_ajax_queue = load_pickle("ms_ajax_queue.p") log( "debug", "Loaded {} entries into ms_ajax_queue".format( len(metasmoke.Metasmoke.ms_ajax_queue))) if has_pickle("seSiteIds.p"): with GlobalVars.site_id_dict_lock: (GlobalVars.site_id_dict_timestamp, GlobalVars.site_id_dict_issues_into_chat_timestamp, GlobalVars.site_id_dict) = load_pickle("seSiteIds.p", encoding='utf-8') fill_site_id_dict_by_id_from_site_id_dict() if has_pickle("recentlyScannedPosts.p"): with GlobalVars.recently_scanned_posts_lock: GlobalVars.recently_scanned_posts = load_pickle( "recentlyScannedPosts.p", encoding='utf-8') blacklists.load_blacklists()
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: chatcommunicate.tell_rooms_with("debug", message['message']) elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body, 'owner': {'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url}, 'site': post_data.site, 'is_answer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count}) spamhandling.handle_spam(post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log --pretty=format:"%H" -n 1').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md)) if line == '\n': i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if line in seen: i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) post = classes.Post(api_response=post_data.as_dict) scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(post) if scan_spam: why_append = u"This post would have also been caught for: " + \ u", ".join(scan_reasons).capitalize() + "\n" + scan_why else: why_append = u"This post would not have been caught otherwise." # Add user to blacklist *after* post is scanned if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format( message["report"]["user"], why_append) spamhandling.handle_spam(post=post, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md)) if line == '\n': i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if line in seen: i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log -1 --pretty="%H"').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)