def unblock(msg, room_id): """ Unblocks posting to a room :param msg: :param room_id: :return: A string """ block_room(room_id, msg._client.host, -1) which_room = "globally" if room_id is None else "in room " + room_id unblock_message = "Reports unblocked {}.".format(which_room) tell_rooms(unblock_message, ("debug", "metatavern"), ()) return report
def handle_spam(post, reasons, why): datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.utcnow())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id != "": datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id) try: post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url)) if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): username = "" else: username = post.user_name.strip() Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) offensive_mask = 'offensive title detected' in reasons message = build_message(post, reasons) if offensive_mask: post.title = "(potentially offensive title -- see MS for details)" clean_message = build_message(post, reasons) log('debug', GlobalVars.parser.unescape(message).encode('ascii', errors='replace')) GlobalVars.deletion_watcher.subscribe(post_url) without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,) if set(reasons) - GlobalVars.experimental_reasons == set() and \ not why.startswith("Post manually "): chatcommunicate.tell_rooms(message, ("experimental-all-sites", "experimental-site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: if offensive_mask: chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site), without_roles + ("offensive-mask",), notify_site=post.post_site, report_data=(post_url, poster_url)) chatcommunicate.tell_rooms(clean_message, ("all-sites", "site-" + post.post_site), without_roles + ("no-offensive-mask",), notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all-sites", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except Exception as e: excepthook.uncaught_exception(*sys.exc_info())
def block(msg, block_time, room_id): """ Blocks posts from application for a period of time :param msg: :param block_time: :param room_id: :return: A string """ time_to_block = block_time if 0 < block_time < 14400 else 900 block_room(room_id, msg._client.host, time.time() + time_to_block) which_room = "globally" if room_id is None else "in room " + room_id block_message = "Reports blocked for {} seconds {}.".format(time_to_block, which_room) tell_rooms(block_message, ("debug", "metatavern"), ()) return report
def handle_spam(post, reasons, why): post_url = parsing.to_protocol_relative( parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative( parsing.user_url_to_shortlink(post.user_url)) reason = ", ".join(reasons[:5]) if len(reasons) > 5: reason += ", +{} more".format(len(reasons) - 5) reason = reason[:1].upper() + reason[ 1:] # reason is capitalised, unlike the entries of reasons list shortened_site = post.post_site.replace( "stackexchange.com", "SE") # site.stackexchange.com -> site.SE datahandling.append_to_latest_questions( post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post( (post.post_id, post.post_site, datetime.now())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id is not "": datahandling.add_post_site_id_link( (post.post_id, post.post_site, "answer"), post.parent.post_id) try: post._title = parsing.escape_special_chars_in_title(post.title) if post.is_answer: # If the post is an answer type post, the 'title' is going to be blank, so when posting the # message contents we need to set the post title to the *parent* title, so the message in the # chat is properly constructed with parent title instead. This will make things 'print' # in a proper way in chat messages. sanitized_title = regex.sub('(https?://|\n)', '', post.parent.title) else: sanitized_title = regex.sub('(https?://|\n)', '', post.title) sanitized_title = regex.sub(r'([\]*`])', r'\\\1', sanitized_title).replace('\n', u'\u23CE') prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]" if GlobalVars.metasmoke_key: prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + \ post_url + ") ]" else: prefix_ms = prefix if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): s = u" {}: [{}]({}) by a deleted user on `{}`".format( reason, sanitized_title.strip(), post_url, shortened_site) username = "" else: s = u" {}: [{}]({}) by [{}]({}) on `{}`".format( reason, sanitized_title.strip(), post_url, post.user_name.strip(), poster_url, shortened_site) username = post.user_name.strip() Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace')) datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title) message = prefix_ms + s if len(message) > 500: message = (prefix + s)[:500] without_roles = tuple( "no-" + reason for reason in reasons) + ("site-no-" + post.post_site, ) if set(reason) & GlobalVars.experimental_reasons == {}: chatcommunicate.tell_rooms(message, ("experimental"), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except: exc_type, exc_obj, exc_tb = sys.exc_info() excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_spam(post, reasons, why): post_url = parsing.to_protocol_relative( parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative( parsing.user_url_to_shortlink(post.user_url)) shortened_site = post.post_site.replace( "stackexchange.com", "SE") # site.stackexchange.com -> site.SE datahandling.append_to_latest_questions( post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post( (post.post_id, post.post_site, datetime.now())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id is not "": datahandling.add_post_site_id_link( (post.post_id, post.post_site, "answer"), post.parent.post_id) try: # If the post is an answer type post, the 'title' is going to be blank, so when posting the # message contents we need to set the post title to the *parent* title, so the message in the # chat is properly constructed with parent title instead. This will make things 'print' # in a proper way in chat messages. sanitized_title = parsing.sanitize_title( post.title if not post.is_answer else post.parent.title) prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]" if GlobalVars.metasmoke_key: prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/uid/{}/{}) ]".format( api_parameter_from_link(post_url), post.post_id) else: prefix_ms = prefix # We'll insert reason list later if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): s = u" {{}}: [{}]({}) by a deleted user on `{}`".format( sanitized_title, post_url, shortened_site) username = "" else: s = u" {{}}: [{}]({}) by [{}]({}) on `{}`".format( sanitized_title, post_url, post.user_name.strip(), poster_url, shortened_site) username = post.user_name.strip() Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace')) GlobalVars.deletion_watcher.subscribe(post_url) reason = message = None for reason_count in range(5, 2, -1): # Try 5 reasons, then 4, then 3 reason = ", ".join(reasons[:reason_count]) if len(reasons) > reason_count: reason += ", +{} more".format(len(reasons) - reason_count) reason = reason[:1].upper() + reason[ 1:] # reason is capitalised, unlike the entries of reasons list message = prefix_ms + s.format(reason) # Insert reason list if len(message) <= 500: break # Problem solved, stop attempting s = s.format(reason) # Later code needs this variable if len(message) > 500: message = (prefix_ms + s)[:500] # Truncate directly and keep MS link without_roles = tuple( "no-" + reason for reason in reasons) + ("site-no-" + post.post_site, ) if set(reasons) - GlobalVars.experimental_reasons == set(): chatcommunicate.tell_rooms(message, ("experimental", ), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except: exc_type, exc_obj, exc_tb = sys.exc_info() excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_websocket_data(data): if "message" not in data: if "type" in data and data['type'] == "reject_subscription": log( 'error', "MS WebSocket subscription was rejected. Check your MS key." ) raise ConnectionError("MS WebSocket connection rejected") return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"], ), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with( "debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with( "debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] recent_commits = sp.check_output( ["git", "log", "-50", "--pretty=%H"]).decode('utf-8').strip().split('\n') if c["commit_sha"] in recent_commits: return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do if c["status"] == "success": if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \ c["commit_message"].startswith("Auto "): s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart_blacklists) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart_findspam) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") exit_mode("pull_update") else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: exit_mode("shutdown") elif "domain_whitelist" in message: if message["domain_whitelist"] == "refresh": metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return # Same rev, nothing to do if c["status"] == "success": if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \ c["commit_message"].startswith("Auto "): s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") exit_mode("pull_update") else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: exit_mode("shutdown")
def handle_spam(post, reasons, why): post_url = parsing.to_protocol_relative( parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative( parsing.user_url_to_shortlink(post.user_url)) shortened_site = post.post_site.replace( "stackexchange.com", "SE") # site.stackexchange.com -> site.SE datahandling.append_to_latest_questions( post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post( (post.post_id, post.post_site, datetime.now())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id is not "": datahandling.add_post_site_id_link( (post.post_id, post.post_site, "answer"), post.parent.post_id) if GlobalVars.reason_weights or GlobalVars.metasmoke_key: reason_weight = sum_weight(reasons) if reason_weight >= 1000: reason_weight_s = " (**{:,}**)".format(reason_weight) else: reason_weight_s = " ({:,})".format(reason_weight) else: # No reason weight if neither cache nor MS reason_weight_s = "" try: # If the post is an answer type post, the 'title' is going to be blank, so when posting the # message contents we need to set the post title to the *parent* title, so the message in the # chat is properly constructed with parent title instead. This will make things 'print' # in a proper way in chat messages. sanitized_title = parsing.sanitize_title( post.title if not post.is_answer else post.parent.title) sanitized_title = escape_format(sanitized_title).strip() prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]" if GlobalVars.metasmoke_key: prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS]({}) ]".format( to_metasmoke_link(post_url, protocol=False)) else: prefix_ms = prefix # We'll insert reason list later edited = '' if not post.edited else ' \u270F\uFE0F' if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): s = " {{}}{}: [{}]({}){} by a deleted user on `{}`".format( reason_weight_s, sanitized_title, post_url, edited, shortened_site) username = "" else: username = post.user_name.strip() escaped_username = escape_format(parsing.escape_markdown(username)) s = " {{}}{}: [{}]({}){} by [{}]({}) on `{}`".format( reason_weight_s, sanitized_title, post_url, edited, escaped_username, poster_url, shortened_site) Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace')) GlobalVars.deletion_watcher.subscribe(post_url) reason = message = None for reason_count in range( 5, 0, -1): # Try 5 reasons and all the way down to 1 reason = ", ".join(reasons[:reason_count]) if len(reasons) > reason_count: reason += ", +{} more".format(len(reasons) - reason_count) reason = reason.capitalize() message = prefix_ms + s.format(reason) # Insert reason list if len(message) <= 500: break # Problem solved, stop attempting s = s.format(reason) # Later code needs this variable if len(message) > 500: message = (prefix_ms + s)[:500] # Truncate directly and keep MS link without_roles = tuple(["no-" + reason for reason in reasons ]) + ("site-no-" + post.post_site, ) if set(reasons) - GlobalVars.experimental_reasons == set() and \ not why.startswith("Post manually "): chatcommunicate.tell_rooms(message, ("experimental", ), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except Exception as e: excepthook.uncaught_exception(*sys.exc_info())
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"], ), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], "the metasmoke API", None, "the metasmoke API") return post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) post = classes.Post(api_response=post_data.as_dict) scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam( post) if scan_spam: why_append = u"This post would have also been caught for: " + \ u", ".join(scan_reasons).capitalize() + "\n" + scan_why else: why_append = u"This post would not have been caught otherwise." # Add user to blacklist *after* post is scanned if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format( message["report"]["user"], why_append) spamhandling.handle_spam( post=post, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with( "debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with( "debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output( ["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"], ), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: datahandling.remove_blacklisted_user( message['unblacklist']['uid']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post( message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format( message["report"]["user"]) postobj = classes.Post( api_response={ 'title': post_data.title, 'body': post_data.body, 'owner': { 'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url }, 'site': post_data.site, 'is_answer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count }) spamhandling.handle_spam( post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log --pretty=format:"%H" -n 1').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed( GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob( 'blacklisted_*.txt' ): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append( "DOS line ending at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append( "No newline at end of `{0}` in {1}" .format(bl_file, commit_md)) if line == '\n': i.append( "Blank line at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if line in seen: i.append( "Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format( line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() chatcommunicate.tell_rooms_with( "debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") chatcommunicate.tell_rooms_with( "debug", ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen( 'git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def handle_spam(post, reasons, why): post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url)) reason = ", ".join(reasons[:5]) if len(reasons) > 5: reason += ", +{} more".format(len(reasons) - 5) reason = reason[:1].upper() + reason[1:] # reason is capitalised, unlike the entries of reasons list shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id is not "": datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id) try: post._title = parsing.escape_special_chars_in_title(post.title) if post.is_answer: # If the post is an answer type post, the 'title' is going to be blank, so when posting the # message contents we need to set the post title to the *parent* title, so the message in the # chat is properly constructed with parent title instead. This will make things 'print' # in a proper way in chat messages. sanitized_title = regex.sub('(https?://|\n)', '', post.parent.title) else: sanitized_title = regex.sub('(https?://|\n)', '', post.title) sanitized_title = regex.sub(r'([\]*`])', r'\\\1', sanitized_title).replace('\n', u'\u23CE') prefix = u"[ [SmokeDetector](//goo.gl/eLDYqh) ]" if GlobalVars.metasmoke_key: prefix_ms = u"[ [SmokeDetector](//goo.gl/eLDYqh) | [MS](//m.erwaysoftware.com/posts/by-url?url=" + \ post_url + ") ]" else: prefix_ms = prefix if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): s = u" {}: [{}]({}) by a deleted user on `{}`".format(reason, sanitized_title.strip(), post_url, shortened_site) username = "" else: s = u" {}: [{}]({}) by [{}]({}) on `{}`".format(reason, sanitized_title.strip(), post_url, post.user_name.strip(), poster_url, shortened_site) username = post.user_name.strip() Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace')) datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title) message = prefix_ms + s if len(message) > 500: message = (prefix + s)[:500] without_roles = tuple("no-" + reason for reason in reasons) if set(reason) & GlobalVars.experimental_reasons == {}: chatcommunicate.tell_rooms(message, ("experimental"), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except: exc_type, exc_obj, exc_tb = sys.exc_info() excepthook.uncaught_exception(exc_type, exc_obj, exc_tb)
def handle_spam(post, reasons, why): post_url = parsing.to_protocol_relative(parsing.url_to_shortlink(post.post_url)) poster_url = parsing.to_protocol_relative(parsing.user_url_to_shortlink(post.user_url)) shortened_site = post.post_site.replace("stackexchange.com", "SE") # site.stackexchange.com -> site.SE datahandling.append_to_latest_questions(post.post_site, post.post_id, post.title if not post.is_answer else "") if len(reasons) == 1 and ("all-caps title" in reasons or "repeating characters in title" in reasons or "repeating characters in body" in reasons or "repeating characters in answer" in reasons or "repeating words in title" in reasons or "repeating words in body" in reasons or "repeating words in answer" in reasons): datahandling.add_auto_ignored_post((post.post_id, post.post_site, datetime.now())) if why is not None and why != "": datahandling.add_why(post.post_site, post.post_id, why) if post.is_answer and post.post_id is not None and post.post_id is not "": datahandling.add_post_site_id_link((post.post_id, post.post_site, "answer"), post.parent.post_id) if GlobalVars.reason_weights or GlobalVars.metasmoke_key: reason_weight = sum_weight(reasons) if reason_weight >= 1000: reason_weight_s = " (**{}**)".format(reason_weight) else: reason_weight_s = " ({})".format(reason_weight) else: # No reason weight if neither cache nor MS reason_weight_s = "" try: # If the post is an answer type post, the 'title' is going to be blank, so when posting the # message contents we need to set the post title to the *parent* title, so the message in the # chat is properly constructed with parent title instead. This will make things 'print' # in a proper way in chat messages. sanitized_title = parsing.sanitize_title(post.title if not post.is_answer else post.parent.title) sanitized_title = escape_format(sanitized_title).strip() prefix = u"[ [SmokeDetector](//git.io/vyDZv) ]" if GlobalVars.metasmoke_key: prefix_ms = u"[ [SmokeDetector](//git.io/vyDZv) | [MS]({}) ]".format( to_metasmoke_link(post_url, protocol=False)) else: prefix_ms = prefix # We'll insert reason list later edited = '' if not post.edited else ' \u270F\uFE0F' if not post.user_name.strip() or (not poster_url or poster_url.strip() == ""): s = " {{}}{}: [{}]({}){} by a deleted user on `{}`".format( reason_weight_s, sanitized_title, post_url, edited, shortened_site) username = "" else: username = post.user_name.strip() escaped_username = escape_format(parsing.escape_markdown(username)) s = " {{}}{}: [{}]({}){} by [{}]({}) on `{}`".format( reason_weight_s, sanitized_title, post_url, edited, escaped_username, poster_url, shortened_site) Tasks.do(metasmoke.Metasmoke.send_stats_on_post, post.title_ignore_type, post_url, reasons, post.body, username, post.user_link, why, post.owner_rep, post.post_score, post.up_vote_count, post.down_vote_count) log('debug', GlobalVars.parser.unescape(s).encode('ascii', errors='replace')) GlobalVars.deletion_watcher.subscribe(post_url) reason = message = None for reason_count in range(5, 0, -1): # Try 5 reasons and all the way down to 1 reason = ", ".join(reasons[:reason_count]) if len(reasons) > reason_count: reason += ", +{} more".format(len(reasons) - reason_count) reason = reason.capitalize() message = prefix_ms + s.format(reason) # Insert reason list if len(message) <= 500: break # Problem solved, stop attempting s = s.format(reason) # Later code needs this variable if len(message) > 500: message = (prefix_ms + s)[:500] # Truncate directly and keep MS link without_roles = tuple(["no-" + reason for reason in reasons]) + ("site-no-" + post.post_site,) if set(reasons) - GlobalVars.experimental_reasons == set() and \ not why.startswith("Post manually "): chatcommunicate.tell_rooms(message, ("experimental",), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) else: chatcommunicate.tell_rooms(message, ("all", "site-" + post.post_site), without_roles, notify_site=post.post_site, report_data=(post_url, poster_url)) except Exception as e: excepthook.uncaught_exception(*sys.exc_info())