def subscribe(self, post_url=None, hostname=None, site_id=None, question_id=None, pickle=True, timeout=DEFAULT_TIMEOUT, max_time=None, from_time=None): if GlobalVars.no_edit_watcher: return if post_url and not ((hostname or site_id) and question_id): post_id, hostname, post_type = fetch_post_id_and_site_from_url(post_url) if post_type == "answer": question_id = datahandling.get_post_site_id_link((post_id, hostname, post_type)) if question_id is None: log("warning", "Unable to get question ID when subscribing to: hostname: " "{} :: post ID:{} when subscribing to {}".format(hostname, post_id, post_url)) return else: question_id = post_id if post_type != "question": log("warning", "tried to edit-watch non-question: hostname: " "{} :: post ID:{} when subscribing to {}".format(hostname, question_id, post_url)) return if not site_id or not hostname: with GlobalVars.site_id_dict_lock: if not site_id and hostname: site_id = GlobalVars.site_id_dict.get(hostname) if site_id and not hostname: hostname = GlobalVars.site_id_dict_by_id.get(site_id) if not site_id or not hostname: log("warning", "unable to determine a valid site ID or hostname when subscribing to question ID " "{}:: site_id:{}:: hostname:{}:: post_url:{}".format(question_id, site_id, hostname, post_url)) return question_ids = question_id if type(question_ids) != list: question_ids = [question_id] now = time.time() if from_time: now = from_time if not max_time: max_time = now + timeout updated = None to_subscribe = [] with self.posts_lock: for question_id in question_ids: action = "{}-question-{}".format(site_id, question_id) if action not in self.posts: self.posts[action] = (site_id, hostname, question_id, max_time) to_subscribe.append(action) else: old_max_time = self.posts[action][2] if max_time > old_max_time: self.posts[action] = (site_id, hostname, question_id, max_time) elif updated is None: updated = False for action in to_subscribe: print('scheduling subscription to action:', action) Tasks.do(self._subscribe, action) if updated and pickle: self._schedule_save()
def subscribe(self, post_url, callback=None, pickle=True, timeout=None): post_id, post_site, post_type = fetch_post_id_and_site_from_url( post_url) if post_site not in GlobalVars.site_id_dict: log( "warning", "unknown site {} when subscribing to {}".format( post_site, post_url)) return if post_type == "answer": question_id = datahandling.get_post_site_id_link( (post_id, post_site, post_type)) if question_id is None: return else: question_id = post_id site_id = GlobalVars.site_id_dict[post_site] action = "{}-question-{}".format(site_id, question_id) max_time = (time.time() + timeout) if timeout else None if action in self.posts and callback: _, _, _, _, callbacks = self.posts[action] callbacks.append((callback, max_time)) else: self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else []) self.socket.send(action) if pickle: Tasks.do(self._save)
def api_get_post(post_url): d = fetch_post_id_and_site_from_url(post_url) if d is None: return None post_id, site, post_type = d if post_type == "answer": api_filter = "!)Q7pIaQiHdpLxolL4GBob2gX" req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() else: assert post_type == "question" api_filter = "!5-dwy(WuZ4NTyJamA*kR8slbR(yU(eC-1U_sEK" req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() if 'items' not in resp_json or len(resp_json['items']) == 0: return False item = resp_json['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = url_to_shortlink(item['link']) post_data.post_type = post_type h = HTMLParser.HTMLParser() post_data.title = h.unescape(item['title']) if 'owner' in item and 'owner' is not None: post_data.owner_name = item['owner']['display_name'] post_data.owner_url = item['owner']['link'] post_data.site = site post_data.body = item['body'] return post_data
def api_get_post(post_url): GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = parsing.fetch_post_id_and_site_from_url(post_url) if d is None: GlobalVars.api_request_lock.release() return None post_id, site, post_type = d if post_type == "answer": api_filter = r"!FdmhxNRjn0vYtGOu3FfS5xSwvL" else: assert post_type == "question" api_filter = r"!DEPw4-PqDduRmCwMBNAxrCdSZl81364qitC3TebCzqyF4-y*r2L" request_url = "https://api.stackexchange.com/2.2/{}s/{}".format( post_type, post_id) params = { 'filter': api_filter, 'key': 'IAkbitmze4B8KpacUfLqkw((', 'site': site } response = requests.get(request_url, params=params).json() if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if 'items' not in response or len(response['items']) == 0: GlobalVars.api_request_lock.release() return False GlobalVars.api_request_lock.release() item = response['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = parsing.url_to_shortlink(item['link']) post_data.post_type = post_type post_data.title = html.unescape(item['title']) if 'owner' in item and 'link' in item['owner']: post_data.owner_name = html.unescape(item['owner']['display_name']) post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] else: post_data.owner_name = "" post_data.owner_url = "" post_data.owner_rep = 1 post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] post_data.creation_date = item['creation_date'] try: post_data.last_edit_date = item['last_edit_date'] except KeyError: post_data.last_edit_date = post_data.creation_date # Key not present = not edited if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def api_get_post(post_url): d = fetch_post_id_and_site_from_url(post_url) if d is None: return None post_id, site, post_type = d if post_type == "answer": api_filter = "!1zSl_EE)(nuF4Xn(2sDLC" req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() else: assert post_type == "question" api_filter = "!gB6tXYzgnc3pG)x0n*03eR9*kZWXReH54Qb" req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() if 'items' not in resp_json or len(resp_json['items']) == 0: return False item = resp_json['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = url_to_shortlink(item['link']) post_data.post_type = post_type h = HTMLParser.HTMLParser() post_data.title = h.unescape(item['title']) if 'owner' in item and 'owner' is not None: post_data.owner_name = item['owner']['display_name'] post_data.owner_url = item['owner']['link'] post_data.site = site return post_data
def subscribe(self, post_url, callback=None, pickle=True, timeout=None): post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url) if post_site not in GlobalVars.site_id_dict: log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url)) return if post_type == "answer": question_id = datahandling.get_post_site_id_link((post_id, post_site, post_type)) if question_id is None: return else: question_id = post_id site_id = GlobalVars.site_id_dict[post_site] action = "{}-question-{}".format(site_id, question_id) max_time = (time.time() + timeout) if timeout else None if action not in self.posts: self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else []) try: self.socket.send(action) except websocket.WebSocketException: log('error', 'DeletionWatcher failed on sending {}'.format(action)) elif callback: _, _, _, _, callbacks = self.posts[action] callbacks.append((callback, max_time)) else: return if pickle: Tasks.do(self._save)
def api_get_post(post_url): GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = parsing.fetch_post_id_and_site_from_url(post_url) if d is None: GlobalVars.api_request_lock.release() return None post_id, site, post_type = d if post_type == "answer": api_filter = r"!FdmhxNRjn0vYtGOu3FfS5xSwvL" else: assert post_type == "question" api_filter = r"!DEPw4-PqDduRmCwMBNAxrCdSZl81364qitC3TebCzqyF4-y*r2L" request_url = "https://api.stackexchange.com/2.2/{}s/{}".format(post_type, post_id) params = { 'filter': api_filter, 'key': 'IAkbitmze4B8KpacUfLqkw((', 'site': site } response = requests.get(request_url, params=params).json() if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if 'items' not in response or len(response['items']) == 0: GlobalVars.api_request_lock.release() return False GlobalVars.api_request_lock.release() item = response['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = parsing.url_to_shortlink(item['link']) post_data.post_type = post_type post_data.title = html.unescape(item['title']) if 'owner' in item and 'link' in item['owner']: post_data.owner_name = html.unescape(item['owner']['display_name']) post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] else: post_data.owner_name = "" post_data.owner_url = "" post_data.owner_rep = 1 post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] post_data.creation_date = item['creation_date'] try: post_data.last_edit_date = item['last_edit_date'] except KeyError: post_data.last_edit_date = post_data.creation_date # Key not present = not edited if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def callback(room=room, msg=msg_pings): post = fetch_post_id_and_site_from_url( report_data[0])[0:2] if not datahandling.is_false_positive( post) and not datahandling.is_ignored_post( post): _msg_queue.put((room, msg, report_data))
def check_if_report_was_deleted(self, post_url, message): post_site_id = fetch_post_id_and_site_from_url(post_url) was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 1200) if was_report_deleted: try: message.delete() except: pass
def check_if_report_was_deleted(self, post_url, message): post_site_id = fetch_post_id_and_site_from_url(post_url) was_report_deleted = self.check_websocket_for_deletion( post_site_id, post_url, 1200) if was_report_deleted: try: message.delete() except: pass
def post_message_if_not_deleted(self, message_text, room, report_data): post_url = report_data[0] post_site_id = fetch_post_id_and_site_from_url(post_url) was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 300) if not was_report_deleted and not datahandling.is_false_positive(post_site_id[0:2]) and not \ datahandling.is_ignored_post(post_site_id[0:2]): chatcommunicate._msg_queue.put((room, message_text, report_data))
def post_message_if_not_deleted(self, message_text, room, report_data): post_url = report_data[0] post_site_id = fetch_post_id_and_site_from_url(post_url) was_report_deleted = self.check_websocket_for_deletion( post_site_id, post_url, 300) if not was_report_deleted and not datahandling.is_false_positive(post_site_id[0:2]) and not \ datahandling.is_ignored_post(post_site_id[0:2]): chatcommunicate._msg_queue.put((room, message_text, report_data))
def api_get_post(post_url): GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = parsing.fetch_post_id_and_site_from_url(post_url) if d is None: GlobalVars.api_request_lock.release() return None post_id, site, post_type = d if post_type == "answer": api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT" else: assert post_type == "question" api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat" request_url = "http://api.stackexchange.com/2.2/{type}s/{post_id}?site={site}&filter={api_filter}&" \ "key=IAkbitmze4B8KpacUfLqkw((".format(type=post_type, post_id=post_id, site=site, api_filter=api_filter) response = requests.get(request_url).json() if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if 'items' not in response or len(response['items']) == 0: GlobalVars.api_request_lock.release() return False GlobalVars.api_request_lock.release() item = response['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = parsing.url_to_shortlink(item['link']) post_data.post_type = post_type h = HTMLParser.HTMLParser() post_data.title = h.unescape(item['title']) if 'owner' in item and 'link' in item['owner']: post_data.owner_name = h.unescape(item['owner']['display_name']) post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] else: post_data.owner_name = "" post_data.owner_url = "" post_data.owner_rep = 1 post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def api_get_post(post_url): GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = parsing.fetch_post_id_and_site_from_url(post_url) if d is None: GlobalVars.api_request_lock.release() return None post_id, site, post_type = d if post_type == "answer": api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT" else: assert post_type == "question" api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat" request_url = "http://api.stackexchange.com/2.2/{type}s/{post_id}?site={site}&filter={api_filter}&" \ "key=IAkbitmze4B8KpacUfLqkw((".format(type=post_type, post_id=post_id, site=site, api_filter=api_filter) response = requests.get(request_url).json() if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if 'items' not in response or len(response['items']) == 0: GlobalVars.api_request_lock.release() return False GlobalVars.api_request_lock.release() item = response['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = parsing.url_to_shortlink(item['link']) post_data.post_type = post_type post_data.title = html.unescape(item['title']) if 'owner' in item and 'link' in item['owner']: post_data.owner_name = html.unescape(item['owner']['display_name']) post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] else: post_data.owner_name = "" post_data.owner_url = "" post_data.owner_rep = 1 post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def api_get_post(post_url): # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = fetch_post_id_and_site_from_url(post_url) if d is None: return None post_id, site, post_type = d if post_type == "answer": api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT" req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() else: assert post_type == "question" api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat" req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() if "backoff" in resp_json: if GlobalVars.api_backoff_time < time.time() + resp_json["backoff"]: GlobalVars.api_backoff_time = time.time() + resp_json["backoff"] if 'items' not in resp_json or len(resp_json['items']) == 0: return False item = resp_json['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = url_to_shortlink(item['link']) post_data.post_type = post_type h = HTMLParser.HTMLParser() post_data.title = h.unescape(item['title']) if 'owner' in item and 'owner' is not None: post_data.owner_name = item['owner']['display_name'] post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def api_get_post(post_url): # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = fetch_post_id_and_site_from_url(post_url) if d is None: return None post_id, site, post_type = d if post_type == "answer": api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT" req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() else: assert post_type == "question" api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat" req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\ "?site=" + site + "&filter=" + api_filter +\ "&key=IAkbitmze4B8KpacUfLqkw((" resp_json = requests.get(req_url).json() if "backoff" in resp_json: if GlobalVars.api_backoff_time < time.time() + resp_json["backoff"]: GlobalVars.api_backoff_time = time.time() + resp_json["backoff"] if 'items' not in resp_json or len(resp_json['items']) == 0: return False item = resp_json['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = url_to_shortlink(item['link']) post_data.post_type = post_type h = HTMLParser.HTMLParser() post_data.title = h.unescape(item['title']) if 'owner' in item and 'owner' is not None: post_data.owner_name = item['owner']['display_name'] post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] post_data.site = site post_data.body = item['body'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] return post_data
def subscribe(self, post_url, callback=None, pickle=True, timeout=None): if GlobalVars.no_deletion_watcher: return post_id, post_site, post_type = fetch_post_id_and_site_from_url( post_url) if post_site not in GlobalVars.site_id_dict: log( "warning", "unknown site {} when subscribing to {}".format( post_site, post_url)) return if post_type == "answer": question_id = datahandling.get_post_site_id_link( (post_id, post_site, post_type)) if question_id is None: return else: question_id = post_id site_id = GlobalVars.site_id_dict[post_site] action = "{}-question-{}".format(site_id, question_id) max_time = (time.time() + timeout) if timeout else None if action not in self.posts: self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else []) try: self.socket.send(action) except websocket.WebSocketException: log('error', 'DeletionWatcher failed on sending {}'.format(action)) elif callback: _, _, _, _, callbacks = self.posts[action] callbacks.append((callback, max_time)) else: return if pickle: Tasks.do(self._save)
def subscribe(self, post_url, callback=None, pickle=True, timeout=None): if GlobalVars.no_deletion_watcher: return post_id, post_site, post_type = fetch_post_id_and_site_from_url( post_url) with GlobalVars.site_id_dict_lock: site_id = GlobalVars.site_id_dict.get(post_site, None) if not site_id: log( "warning", "unknown site {} when subscribing to {}".format( post_site, post_url)) return if post_type == "answer": question_id = datahandling.get_post_site_id_link( (post_id, post_site, post_type)) if question_id is None: return else: question_id = post_id action = "{}-question-{}".format(site_id, question_id) max_time = (time.time() + timeout) if timeout else None with self.posts_lock: if action not in self.posts: self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else []) Tasks.do(self._subscribe, action) elif callback: _, _, _, _, callbacks = self.posts[action] callbacks.append((callback, max_time)) else: return if pickle: self._schedule_save()
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message['message']) elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: datahandling.add_blacklisted_user( (message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post( message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format( message["report"]["user"]) postobj = classes.Post( api_response={ 'title': post_data.title, 'body': post_data.body, 'owner': { 'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url }, 'site': post_data.site, 'IsAnswer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count }) spamhandling.handle_spam( post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log --pretty=format:"%H" -n 1').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed( GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob( 'blacklisted_*.txt' ): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append( "DOS line ending at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append( "No newline at end of `{0}` in {1}" .format(bl_file, commit_md)) if line == '\n': i.append( "Blank line at `{0}:{1}` in {2}" .format( bl_file, lineno, commit_md)) if line in seen: i.append( "Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format( line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() GlobalVars.charcoal_hq.send_message( "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") GlobalVars.charcoal_hq.send_message( ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen( 'git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) GlobalVars.charcoal_hq.send_message(s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) # noinspection PyUnboundLocalVariable GlobalVars.charcoal_hq.send_message(s) elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def init_websocket(self): try: GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host) GlobalVars.metasmoke_ws.send(json.dumps({"command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\",\"key\":\"" + GlobalVars.metasmoke_key + "\"}"})) while True: a = GlobalVars.metasmoke_ws.recv() print(a) try: data = json.loads(a) if "message" in data: message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message['message']) elif "blacklist" in message: datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: continue if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) and not datahandling.is_false_positive((post_data.post_id, post_data.site)): continue user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) spamhandling.handle_spam(title=post_data.title, body=post_data.body, poster=post_data.owner_name, site=post_data.site, post_url=post_data.post_url, poster_url=post_data.owner_url, post_id=post_data.post_id, reasons=["Manually reported " + post_data.post_type], is_answer=post_data.post_type == "answer", why=why, owner_rep=post_data.owner_rep, post_score=post_data.score, up_vote_count=post_data.up_vote_count, down_vote_count=post_data.down_vote_count, question_id=post_data.question_id) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha)) time.sleep(2) os._exit(3) else: GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)) elif c["status"] == "failure": GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} failed.".format(ci_link=c["ci_url"], commit_sha=sha)) except Exception, e: GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host) GlobalVars.metasmoke_ws.send(json.dumps({"command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\"}"})) print e try: exc_info = sys.exc_info() traceback.print_exception(*exc_info) except: print "meh" except: print "Couldn't bind to MS websocket"
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message['message']) elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body, 'owner': {'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url}, 'site': post_data.site, 'IsAnswer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count}) spamhandling.handle_spam(post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) GlobalVars.charcoal_hq.send_message(s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) # noinspection PyUnboundLocalVariable GlobalVars.charcoal_hq.send_message(s)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def api_get_post(post_url): GlobalVars.api_request_lock.acquire() # Respect backoff, if we were given one if GlobalVars.api_backoff_time > time.time(): time.sleep(GlobalVars.api_backoff_time - time.time() + 2) d = parsing.fetch_post_id_and_site_from_url(post_url) if d is None: GlobalVars.api_request_lock.release() return None post_id, site, post_type = d if post_type == "answer": api_filter = r"!4z6S)cPO)zvpuDWsWTAUW(kaV6K6thsqi1tlYa" elif post_type == "question": api_filter = r"!m)9.UaQrI5-DZXtlTpWhv2HroYRgS3dPhv.2vxV7fpGT*27rEHM.BKV1" else: raise ValueError("Unknown post type: {}".format(post_type)) request_url = "https://api.stackexchange.com/2.2/{}s/{}".format( post_type, post_id) params = { 'filter': api_filter, 'key': 'IAkbitmze4B8KpacUfLqkw((', 'site': site } response = requests.get(request_url, params=params).json() if "backoff" in response: if GlobalVars.api_backoff_time < time.time() + response["backoff"]: GlobalVars.api_backoff_time = time.time() + response["backoff"] if 'items' not in response or len(response['items']) == 0: GlobalVars.api_request_lock.release() return False GlobalVars.api_request_lock.release() item = response['items'][0] post_data = PostData() post_data.post_id = post_id post_data.post_url = parsing.url_to_shortlink(item['link']) post_data.post_type = post_type post_data.title = html.unescape(item['title']) if 'owner' in item and 'link' in item['owner']: post_data.owner_name = html.unescape(item['owner']['display_name']) post_data.owner_url = item['owner']['link'] post_data.owner_rep = item['owner']['reputation'] else: post_data.owner_name = "" post_data.owner_url = "" post_data.owner_rep = 1 post_data.site = site post_data.body = item['body'] post_data.body_markdown = item['body_markdown'] post_data.score = item['score'] post_data.up_vote_count = item['up_vote_count'] post_data.down_vote_count = item['down_vote_count'] post_data.creation_date = item['creation_date'] try: post_data.last_edit_date = item['last_edit_date'] except KeyError: post_data.last_edit_date = post_data.creation_date # Key not present = not edited if post_type == "answer": post_data.question_id = item['question_id'] return post_data
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message['message']) elif "blacklist" in message: datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) spamhandling.handle_spam(title=post_data.title, body=post_data.body, poster=post_data.owner_name, site=post_data.site, post_url=post_data.post_url, poster_url=post_data.owner_url, post_id=post_data.post_id, reasons=["Manually reported " + post_data.post_type], is_answer=post_data.post_type == "answer", why=why, owner_rep=post_data.owner_rep, post_score=post_data.score, up_vote_count=post_data.up_vote_count, down_vote_count=post_data.down_vote_count, question_id=post_data.question_id) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) GlobalVars.charcoal_hq.send_message(s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) # noinspection PyUnboundLocalVariable GlobalVars.charcoal_hq.send_message(s)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) post = classes.Post(api_response=post_data.as_dict) scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(post) if scan_spam: why_append = u"This post would have also been caught for: " + \ u", ".join(scan_reasons).capitalize() + "\n" + scan_why else: why_append = u"This post would not have been caught otherwise." # Add user to blacklist *after* post is scanned if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format( message["report"]["user"], why_append) spamhandling.handle_spam(post=post, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md)) if line == '\n': i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if line in seen: i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log -1 --pretty="%H"').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def callback(room=room, msg=msg_pings): post = fetch_post_id_and_site_from_url(report_data[0])[0:2] if not datahandling.is_false_positive(post) and not datahandling.is_ignored_post(post): _msg_queue.put((room, msg, report_data))
def handle_websocket_data(data): if "message" not in data: return message = data["message"] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message(message["message"]) elif "blacklist" in message: datahandling.add_blacklisted_user( (message["blacklist"]["uid"], message["blacklist"]["site"]), "metasmoke", message["blacklist"]["post"], ) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted( post_data.site, post_data.post_id, post_data.title ) and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) spamhandling.handle_spam( title=post_data.title, body=post_data.body, poster=post_data.owner_name, site=post_data.site, post_url=post_data.post_url, poster_url=post_data.owner_url, post_id=post_data.post_id, reasons=["Manually reported " + post_data.post_type], is_answer=post_data.post_type == "answer", why=why, owner_rep=post_data.owner_rep, post_score=post_data.score, up_vote_count=post_data.up_vote_count, down_vote_count=post_data.down_vote_count, question_id=post_data.question_id, ) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], commit_sha=sha ) GlobalVars.charcoal_hq.send_message(s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on {commit_sha} succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) elif c["status"] == "failure": s = "[CI]({ci_link}) on {commit_sha} failed.".format(ci_link=c["ci_url"], commit_sha=sha) # noinspection PyUnboundLocalVariable GlobalVars.charcoal_hq.send_message(s)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if isinstance(message, Iterable): if "message" in message: chatcommunicate.tell_rooms_with("debug", message['message']) elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"]) postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body, 'owner': {'display_name': post_data.owner_name, 'reputation': post_data.owner_rep, 'link': post_data.owner_url}, 'site': post_data.site, 'is_answer': (post_data.post_type == "answer"), 'score': post_data.score, 'link': post_data.post_url, 'question_id': post_data.post_id, 'up_vote_count': post_data.up_vote_count, 'down_vote_count': post_data.down_vote_count}) spamhandling.handle_spam(post=postobj, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log --pretty=format:"%H" -n 1').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \ .format(sha[:7]) i = [] # Currently no issues with backlists for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'): # Check blacklists for issues with open(bl_file, 'r') as lines: seen = dict() for lineno, line in enumerate(lines, 1): if line.endswith('\r\n'): i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if not line.endswith('\n'): i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md)) if line == '\n': i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno, commit_md)) if line in seen: i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}" .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md)) seen[line] = lineno if i == []: # No issues GitManager.pull_remote() load_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: i.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(i)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read(): if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha})"\ " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) time.sleep(2) os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \ "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s) elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"],), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"]["message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with("debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return # Same rev, nothing to do if c["status"] == "success": if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \ c["commit_message"].startswith("Auto "): s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") exit_mode("pull_update") else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: exit_mode("shutdown")
def init_websocket(self): try: GlobalVars.metasmoke_ws = websocket.create_connection( GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host) GlobalVars.metasmoke_ws.send( json.dumps({ "command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\",\"key\":\"" + GlobalVars.metasmoke_key + "\"}" })) while True: a = GlobalVars.metasmoke_ws.recv() print(a) try: data = json.loads(a) if "message" in data: message = data['message'] if isinstance(message, Iterable): if "message" in message: GlobalVars.charcoal_hq.send_message( message['message']) elif "blacklist" in message: datahandling.add_blacklisted_user( (message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post']) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post( post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive( post_site_id[0:2]) elif "report" in message: post_data = apigetpost.api_get_post( message["report"]["post_link"]) if post_data is None or post_data is False: continue if datahandling.has_already_been_posted( post_data.site, post_data.post_id, post_data.title ) and not datahandling.is_false_positive( (post_data.post_id, post_data.site)): continue user = parsing.get_user_from_url( post_data.owner_url) if user is not None: datahandling.add_blacklisted_user( user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n".format( message["report"]["user"]) spamhandling.handle_spam( title=post_data.title, body=post_data.body, poster=post_data.owner_name, site=post_data.site, post_url=post_data.post_url, poster_url=post_data.owner_url, post_id=post_data.post_id, reasons=[ "Manually reported " + post_data.post_type ], is_answer=post_data.post_type == "answer", why=why, owner_rep=post_data.owner_rep, post_score=post_data.score, up_vote_count=post_data.up_vote_count, down_vote_count=post_data.down_vote_count, question_id=post_data.question_id) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] != os.popen( 'git log --pretty=format:"%H" -n 1' ).read(): if c["status"] == "success": if "autopull" in c["commit_message"]: GlobalVars.charcoal_hq.send_message( "[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling..." .format(ci_link=c["ci_url"], commit_sha=sha)) time.sleep(2) os._exit(3) else: GlobalVars.charcoal_hq.send_message( "[CI]({ci_link}) on {commit_sha} succeeded." .format(ci_link=c["ci_url"], commit_sha=sha)) elif c["status"] == "failure": GlobalVars.charcoal_hq.send_message( "[CI]({ci_link}) on {commit_sha} failed." .format(ci_link=c["ci_url"], commit_sha=sha)) except Exception, e: GlobalVars.metasmoke_ws = websocket.create_connection( GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host) GlobalVars.metasmoke_ws.send( json.dumps({ "command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\"}" })) print e try: exc_info = sys.exc_info() traceback.print_exception(*exc_info) except: print "meh" except: print "Couldn't bind to MS websocket"
def handle_websocket_data(data): if "message" not in data: if "type" in data and data['type'] == "reject_subscription": log( 'error', "MS WebSocket subscription was rejected. Check your MS key." ) raise ConnectionError("MS WebSocket connection rejected") return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"], ), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"], True, "the metasmoke API") elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with( "debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with( "debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] recent_commits = sp.check_output( ["git", "log", "-50", "--pretty=%H"]).decode('utf-8').strip().split('\n') if c["commit_sha"] in recent_commits: return # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do if c["status"] == "success": if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \ c["commit_message"].startswith("Auto "): s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart_blacklists) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_branch: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) exit_mode("checkout_deploy") GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart_findspam) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") exit_mode("pull_update") else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: exit_mode("shutdown") elif "domain_whitelist" in message: if message["domain_whitelist"] == "refresh": metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
def handle_websocket_data(data): if "message" not in data: return message = data['message'] if not isinstance(message, Iterable): return if "message" in message: chatcommunicate.tell_rooms_with("metasmoke", message['message']) elif "autoflag_fp" in message: event = message["autoflag_fp"] chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]), ("no-site-" + event["site"], ), notify_site="/autoflag_fp") elif "exit" in message: os._exit(message["exit"]) elif "blacklist" in message: ids = (message['blacklist']['uid'], message['blacklist']['site']) datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post']) datahandling.last_feedbacked = (ids, time.time() + 60) elif "unblacklist" in message: ids = (message['unblacklist']['uid'], message['unblacklist']['site']) datahandling.remove_blacklisted_user(ids) elif "naa" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["naa"]["post_link"]) datahandling.add_ignored_post(post_site_id[0:2]) elif "fp" in message: post_site_id = parsing.fetch_post_id_and_site_from_url( message["fp"]["post_link"]) datahandling.add_false_positive(post_site_id[0:2]) elif "report" in message: import chatcommands # Do it here chatcommands.report_posts([message["report"]["post_link"]], "the metasmoke API", None, "the metasmoke API") return post_data = apigetpost.api_get_post(message["report"]["post_link"]) if post_data is None or post_data is False: return if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \ and not datahandling.is_false_positive((post_data.post_id, post_data.site)): return user = parsing.get_user_from_url(post_data.owner_url) post = classes.Post(api_response=post_data.as_dict) scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam( post) if scan_spam: why_append = u"This post would have also been caught for: " + \ u", ".join(scan_reasons).capitalize() + "\n" + scan_why else: why_append = u"This post would not have been caught otherwise." # Add user to blacklist *after* post is scanned if user is not None: datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url) why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format( message["report"]["user"], why_append) spamhandling.handle_spam( post=post, reasons=["Manually reported " + post_data.post_type], why=why) elif "deploy_updated" in message: return # Disabled sha = message["deploy_updated"]["head_commit"]["id"] if sha != os.popen('git log -1 --pretty="%H"').read(): if "autopull" in message["deploy_updated"]["head_commit"][ "message"]: if only_blacklists_changed(GitManager.get_remote_diff()): commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \ .format(sha[:7], GlobalVars.bot_repo_slug) integrity = blacklist_integrity_check() if len(integrity) == 0: # No issues GitManager.pull_remote() findspam.reload_blacklists() chatcommunicate.tell_rooms_with( "debug", "No code modified in {0}, only blacklists" " reloaded.".format(commit_md)) else: integrity.append("please fix before pulling.") chatcommunicate.tell_rooms_with( "debug", ", ".join(integrity)) elif "commit_status" in message: c = message["commit_status"] sha = c["commit_sha"][:7] if c["commit_sha"] == sp.check_output( ["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip(): return if c["status"] == "success": if "autopull" in c["commit_message"]: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \ "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format( ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) remote_diff = GitManager.get_remote_diff() if only_blacklists_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() findspam.FindSpam.reload_blacklists() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart) elif only_modules_changed(remote_diff): GitManager.pull_remote() if not GlobalVars.on_master: # Restart if HEAD detached log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True) os._exit(8) GlobalVars.reload() reload_modules() chatcommunicate.tell_rooms_with( 'debug', GlobalVars.s_norestart2) else: chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci") os._exit(3) else: s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif c["status"] == "failure": s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \ "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha) chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci") elif "everything_is_broken" in message: if message["everything_is_broken"] is True: os._exit(6)