def subscribe(self, post_url=None, hostname=None, site_id=None, question_id=None,
                  pickle=True, timeout=DEFAULT_TIMEOUT, max_time=None, from_time=None):
        if GlobalVars.no_edit_watcher:
            return
        if post_url and not ((hostname or site_id) and question_id):
            post_id, hostname, post_type = fetch_post_id_and_site_from_url(post_url)
            if post_type == "answer":
                question_id = datahandling.get_post_site_id_link((post_id, hostname, post_type))
                if question_id is None:
                    log("warning", "Unable to get question ID when subscribing to: hostname: "
                                   "{} :: post ID:{} when subscribing to {}".format(hostname, post_id, post_url))
                    return
            else:
                question_id = post_id
            if post_type != "question":
                log("warning", "tried to edit-watch non-question: hostname: "
                               "{} :: post ID:{} when subscribing to {}".format(hostname, question_id, post_url))
                return
        if not site_id or not hostname:
            with GlobalVars.site_id_dict_lock:
                if not site_id and hostname:
                    site_id = GlobalVars.site_id_dict.get(hostname)
                if site_id and not hostname:
                    hostname = GlobalVars.site_id_dict_by_id.get(site_id)
        if not site_id or not hostname:
            log("warning", "unable to determine a valid site ID or hostname when subscribing to question ID "
                           "{}:: site_id:{}::  hostname:{}::  post_url:{}".format(question_id, site_id, hostname,
                                                                                  post_url))
            return

        question_ids = question_id
        if type(question_ids) != list:
            question_ids = [question_id]
        now = time.time()
        if from_time:
            now = from_time
        if not max_time:
            max_time = now + timeout

        updated = None
        to_subscribe = []
        with self.posts_lock:
            for question_id in question_ids:
                action = "{}-question-{}".format(site_id, question_id)
                if action not in self.posts:
                    self.posts[action] = (site_id, hostname, question_id, max_time)
                    to_subscribe.append(action)
                else:
                    old_max_time = self.posts[action][2]
                    if max_time > old_max_time:
                        self.posts[action] = (site_id, hostname, question_id, max_time)
                    elif updated is None:
                        updated = False

        for action in to_subscribe:
            print('scheduling subscription to action:', action)
            Tasks.do(self._subscribe, action)

        if updated and pickle:
            self._schedule_save()
예제 #2
0
    def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
        post_id, post_site, post_type = fetch_post_id_and_site_from_url(
            post_url)

        if post_site not in GlobalVars.site_id_dict:
            log(
                "warning", "unknown site {} when subscribing to {}".format(
                    post_site, post_url))
            return

        if post_type == "answer":
            question_id = datahandling.get_post_site_id_link(
                (post_id, post_site, post_type))

            if question_id is None:
                return
        else:
            question_id = post_id

        site_id = GlobalVars.site_id_dict[post_site]
        action = "{}-question-{}".format(site_id, question_id)
        max_time = (time.time() + timeout) if timeout else None

        if action in self.posts and callback:
            _, _, _, _, callbacks = self.posts[action]
            callbacks.append((callback, max_time))
        else:
            self.posts[action] = (post_id, post_site, post_type, post_url,
                                  [(callback, max_time)] if callback else [])
            self.socket.send(action)

        if pickle:
            Tasks.do(self._save)
예제 #3
0
def api_get_post(post_url):
    d = fetch_post_id_and_site_from_url(post_url)
    if d is None:
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!)Q7pIaQiHdpLxolL4GBob2gX"
        req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\
                  "?site=" + site + "&filter=" + api_filter +\
                  "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    else:
        assert post_type == "question"

        api_filter = "!5-dwy(WuZ4NTyJamA*kR8slbR(yU(eC-1U_sEK"
        req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\
            "?site=" + site + "&filter=" + api_filter +\
            "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    if 'items' not in resp_json or len(resp_json['items']) == 0:
        return False
    item = resp_json['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = url_to_shortlink(item['link'])
    post_data.post_type = post_type
    h = HTMLParser.HTMLParser()
    post_data.title = h.unescape(item['title'])
    if 'owner' in item and 'owner' is not None:
        post_data.owner_name = item['owner']['display_name']
        post_data.owner_url = item['owner']['link']
    post_data.site = site
    post_data.body = item['body']
    return post_data
예제 #4
0
def api_get_post(post_url):
    GlobalVars.api_request_lock.acquire()

    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = parsing.fetch_post_id_and_site_from_url(post_url)
    if d is None:
        GlobalVars.api_request_lock.release()
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = r"!FdmhxNRjn0vYtGOu3FfS5xSwvL"
    else:
        assert post_type == "question"
        api_filter = r"!DEPw4-PqDduRmCwMBNAxrCdSZl81364qitC3TebCzqyF4-y*r2L"

    request_url = "https://api.stackexchange.com/2.2/{}s/{}".format(
        post_type, post_id)
    params = {
        'filter': api_filter,
        'key': 'IAkbitmze4B8KpacUfLqkw((',
        'site': site
    }
    response = requests.get(request_url, params=params).json()
    if "backoff" in response:
        if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
            GlobalVars.api_backoff_time = time.time() + response["backoff"]
    if 'items' not in response or len(response['items']) == 0:
        GlobalVars.api_request_lock.release()
        return False
    GlobalVars.api_request_lock.release()

    item = response['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = parsing.url_to_shortlink(item['link'])
    post_data.post_type = post_type
    post_data.title = html.unescape(item['title'])
    if 'owner' in item and 'link' in item['owner']:
        post_data.owner_name = html.unescape(item['owner']['display_name'])
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    else:
        post_data.owner_name = ""
        post_data.owner_url = ""
        post_data.owner_rep = 1
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    post_data.creation_date = item['creation_date']
    try:
        post_data.last_edit_date = item['last_edit_date']
    except KeyError:
        post_data.last_edit_date = post_data.creation_date  # Key not present = not edited
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #5
0
def api_get_post(post_url):
    d = fetch_post_id_and_site_from_url(post_url)
    if d is None:
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!1zSl_EE)(nuF4Xn(2sDLC"
        req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\
                  "?site=" + site + "&filter=" + api_filter +\
                  "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    else:
        assert post_type == "question"

        api_filter = "!gB6tXYzgnc3pG)x0n*03eR9*kZWXReH54Qb"
        req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\
            "?site=" + site + "&filter=" + api_filter +\
            "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    if 'items' not in resp_json or len(resp_json['items']) == 0:
        return False
    item = resp_json['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = url_to_shortlink(item['link'])
    post_data.post_type = post_type
    h = HTMLParser.HTMLParser()
    post_data.title = h.unescape(item['title'])
    if 'owner' in item and 'owner' is not None:
        post_data.owner_name = item['owner']['display_name']
        post_data.owner_url = item['owner']['link']
    post_data.site = site
    return post_data
예제 #6
0
    def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
        post_id, post_site, post_type = fetch_post_id_and_site_from_url(post_url)

        if post_site not in GlobalVars.site_id_dict:
            log("warning", "unknown site {} when subscribing to {}".format(post_site, post_url))
            return

        if post_type == "answer":
            question_id = datahandling.get_post_site_id_link((post_id, post_site, post_type))

            if question_id is None:
                return
        else:
            question_id = post_id

        site_id = GlobalVars.site_id_dict[post_site]
        action = "{}-question-{}".format(site_id, question_id)
        max_time = (time.time() + timeout) if timeout else None

        if action not in self.posts:
            self.posts[action] = (post_id, post_site, post_type, post_url, [(callback, max_time)] if callback else [])
            try:
                self.socket.send(action)
            except websocket.WebSocketException:
                log('error', 'DeletionWatcher failed on sending {}'.format(action))
        elif callback:
            _, _, _, _, callbacks = self.posts[action]
            callbacks.append((callback, max_time))
        else:
            return

        if pickle:
            Tasks.do(self._save)
예제 #7
0
def api_get_post(post_url):
    GlobalVars.api_request_lock.acquire()

    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = parsing.fetch_post_id_and_site_from_url(post_url)
    if d is None:
        GlobalVars.api_request_lock.release()
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = r"!FdmhxNRjn0vYtGOu3FfS5xSwvL"
    else:
        assert post_type == "question"
        api_filter = r"!DEPw4-PqDduRmCwMBNAxrCdSZl81364qitC3TebCzqyF4-y*r2L"

    request_url = "https://api.stackexchange.com/2.2/{}s/{}".format(post_type, post_id)
    params = {
        'filter': api_filter,
        'key': 'IAkbitmze4B8KpacUfLqkw((',
        'site': site
    }
    response = requests.get(request_url, params=params).json()
    if "backoff" in response:
        if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
            GlobalVars.api_backoff_time = time.time() + response["backoff"]
    if 'items' not in response or len(response['items']) == 0:
        GlobalVars.api_request_lock.release()
        return False
    GlobalVars.api_request_lock.release()

    item = response['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = parsing.url_to_shortlink(item['link'])
    post_data.post_type = post_type
    post_data.title = html.unescape(item['title'])
    if 'owner' in item and 'link' in item['owner']:
        post_data.owner_name = html.unescape(item['owner']['display_name'])
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    else:
        post_data.owner_name = ""
        post_data.owner_url = ""
        post_data.owner_rep = 1
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    post_data.creation_date = item['creation_date']
    try:
        post_data.last_edit_date = item['last_edit_date']
    except KeyError:
        post_data.last_edit_date = post_data.creation_date  # Key not present = not edited
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #8
0
                    def callback(room=room, msg=msg_pings):
                        post = fetch_post_id_and_site_from_url(
                            report_data[0])[0:2]

                        if not datahandling.is_false_positive(
                                post) and not datahandling.is_ignored_post(
                                    post):
                            _msg_queue.put((room, msg, report_data))
    def check_if_report_was_deleted(self, post_url, message):
        post_site_id = fetch_post_id_and_site_from_url(post_url)
        was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 1200)

        if was_report_deleted:
            try:
                message.delete()
            except:
                pass
예제 #10
0
    def check_if_report_was_deleted(self, post_url, message):
        post_site_id = fetch_post_id_and_site_from_url(post_url)
        was_report_deleted = self.check_websocket_for_deletion(
            post_site_id, post_url, 1200)

        if was_report_deleted:
            try:
                message.delete()
            except:
                pass
예제 #11
0
    def post_message_if_not_deleted(self, message_text, room, report_data):
        post_url = report_data[0]

        post_site_id = fetch_post_id_and_site_from_url(post_url)
        was_report_deleted = self.check_websocket_for_deletion(post_site_id, post_url, 300)

        if not was_report_deleted and not datahandling.is_false_positive(post_site_id[0:2]) and not \
                datahandling.is_ignored_post(post_site_id[0:2]):

            chatcommunicate._msg_queue.put((room, message_text, report_data))
예제 #12
0
    def post_message_if_not_deleted(self, message_text, room, report_data):
        post_url = report_data[0]

        post_site_id = fetch_post_id_and_site_from_url(post_url)
        was_report_deleted = self.check_websocket_for_deletion(
            post_site_id, post_url, 300)

        if not was_report_deleted and not datahandling.is_false_positive(post_site_id[0:2]) and not \
                datahandling.is_ignored_post(post_site_id[0:2]):

            chatcommunicate._msg_queue.put((room, message_text, report_data))
예제 #13
0
def api_get_post(post_url):
    GlobalVars.api_request_lock.acquire()

    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = parsing.fetch_post_id_and_site_from_url(post_url)
    if d is None:
        GlobalVars.api_request_lock.release()
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT"
    else:
        assert post_type == "question"
        api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat"

    request_url = "http://api.stackexchange.com/2.2/{type}s/{post_id}?site={site}&filter={api_filter}&" \
                  "key=IAkbitmze4B8KpacUfLqkw((".format(type=post_type, post_id=post_id, site=site,
                                                        api_filter=api_filter)
    response = requests.get(request_url).json()
    if "backoff" in response:
        if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
            GlobalVars.api_backoff_time = time.time() + response["backoff"]
    if 'items' not in response or len(response['items']) == 0:
        GlobalVars.api_request_lock.release()
        return False
    GlobalVars.api_request_lock.release()

    item = response['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = parsing.url_to_shortlink(item['link'])
    post_data.post_type = post_type
    h = HTMLParser.HTMLParser()
    post_data.title = h.unescape(item['title'])
    if 'owner' in item and 'link' in item['owner']:
        post_data.owner_name = h.unescape(item['owner']['display_name'])
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    else:
        post_data.owner_name = ""
        post_data.owner_url = ""
        post_data.owner_rep = 1
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #14
0
def api_get_post(post_url):
    GlobalVars.api_request_lock.acquire()

    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = parsing.fetch_post_id_and_site_from_url(post_url)
    if d is None:
        GlobalVars.api_request_lock.release()
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT"
    else:
        assert post_type == "question"
        api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat"

    request_url = "http://api.stackexchange.com/2.2/{type}s/{post_id}?site={site}&filter={api_filter}&" \
                  "key=IAkbitmze4B8KpacUfLqkw((".format(type=post_type, post_id=post_id, site=site,
                                                        api_filter=api_filter)
    response = requests.get(request_url).json()
    if "backoff" in response:
        if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
            GlobalVars.api_backoff_time = time.time() + response["backoff"]
    if 'items' not in response or len(response['items']) == 0:
        GlobalVars.api_request_lock.release()
        return False
    GlobalVars.api_request_lock.release()

    item = response['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = parsing.url_to_shortlink(item['link'])
    post_data.post_type = post_type
    post_data.title = html.unescape(item['title'])
    if 'owner' in item and 'link' in item['owner']:
        post_data.owner_name = html.unescape(item['owner']['display_name'])
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    else:
        post_data.owner_name = ""
        post_data.owner_url = ""
        post_data.owner_rep = 1
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #15
0
def api_get_post(post_url):
    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = fetch_post_id_and_site_from_url(post_url)
    if d is None:
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT"
        req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\
                  "?site=" + site + "&filter=" + api_filter +\
                  "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    else:
        assert post_type == "question"

        api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat"
        req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\
            "?site=" + site + "&filter=" + api_filter +\
            "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    if "backoff" in resp_json:
        if GlobalVars.api_backoff_time < time.time() + resp_json["backoff"]:
            GlobalVars.api_backoff_time = time.time() + resp_json["backoff"]
    if 'items' not in resp_json or len(resp_json['items']) == 0:
        return False
    item = resp_json['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = url_to_shortlink(item['link'])
    post_data.post_type = post_type
    h = HTMLParser.HTMLParser()
    post_data.title = h.unescape(item['title'])
    if 'owner' in item and 'owner' is not None:
        post_data.owner_name = item['owner']['display_name']
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #16
0
def api_get_post(post_url):
    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = fetch_post_id_and_site_from_url(post_url)
    if d is None:
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = "!FdmhxNQy0ZXsmxUOvWMVSbuktT"
        req_url = "http://api.stackexchange.com/2.2/answers/" + post_id +\
                  "?site=" + site + "&filter=" + api_filter +\
                  "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    else:
        assert post_type == "question"

        api_filter = "!)Ehu.SHRfXhu2eCP4p6wd*Wxyw1XouU5qO83b7X5GQK6ciVat"
        req_url = "http://api.stackexchange.com/2.2/questions/" + post_id +\
            "?site=" + site + "&filter=" + api_filter +\
            "&key=IAkbitmze4B8KpacUfLqkw(("
        resp_json = requests.get(req_url).json()
    if "backoff" in resp_json:
        if GlobalVars.api_backoff_time < time.time() + resp_json["backoff"]:
            GlobalVars.api_backoff_time = time.time() + resp_json["backoff"]
    if 'items' not in resp_json or len(resp_json['items']) == 0:
        return False
    item = resp_json['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = url_to_shortlink(item['link'])
    post_data.post_type = post_type
    h = HTMLParser.HTMLParser()
    post_data.title = h.unescape(item['title'])
    if 'owner' in item and 'owner' is not None:
        post_data.owner_name = item['owner']['display_name']
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    post_data.site = site
    post_data.body = item['body']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    return post_data
예제 #17
0
    def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
        if GlobalVars.no_deletion_watcher:
            return
        post_id, post_site, post_type = fetch_post_id_and_site_from_url(
            post_url)

        if post_site not in GlobalVars.site_id_dict:
            log(
                "warning", "unknown site {} when subscribing to {}".format(
                    post_site, post_url))
            return

        if post_type == "answer":
            question_id = datahandling.get_post_site_id_link(
                (post_id, post_site, post_type))

            if question_id is None:
                return
        else:
            question_id = post_id

        site_id = GlobalVars.site_id_dict[post_site]
        action = "{}-question-{}".format(site_id, question_id)
        max_time = (time.time() + timeout) if timeout else None

        if action not in self.posts:
            self.posts[action] = (post_id, post_site, post_type, post_url,
                                  [(callback, max_time)] if callback else [])
            try:
                self.socket.send(action)
            except websocket.WebSocketException:
                log('error',
                    'DeletionWatcher failed on sending {}'.format(action))
        elif callback:
            _, _, _, _, callbacks = self.posts[action]
            callbacks.append((callback, max_time))
        else:
            return

        if pickle:
            Tasks.do(self._save)
    def subscribe(self, post_url, callback=None, pickle=True, timeout=None):
        if GlobalVars.no_deletion_watcher:
            return
        post_id, post_site, post_type = fetch_post_id_and_site_from_url(
            post_url)

        with GlobalVars.site_id_dict_lock:
            site_id = GlobalVars.site_id_dict.get(post_site, None)
        if not site_id:
            log(
                "warning", "unknown site {} when subscribing to {}".format(
                    post_site, post_url))
            return

        if post_type == "answer":
            question_id = datahandling.get_post_site_id_link(
                (post_id, post_site, post_type))

            if question_id is None:
                return
        else:
            question_id = post_id

        action = "{}-question-{}".format(site_id, question_id)
        max_time = (time.time() + timeout) if timeout else None

        with self.posts_lock:
            if action not in self.posts:
                self.posts[action] = (post_id, post_site, post_type, post_url,
                                      [(callback,
                                        max_time)] if callback else [])
                Tasks.do(self._subscribe, action)
            elif callback:
                _, _, _, _, callbacks = self.posts[action]
                callbacks.append((callback, max_time))
            else:
                return

        if pickle:
            self._schedule_save()
예제 #19
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                GlobalVars.charcoal_hq.send_message(message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user(
                    (message['blacklist']['uid'],
                     message['blacklist']['site']), "metasmoke",
                    message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(
                    message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(
                    message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(
                    message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke",
                                                      post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(
                    message["report"]["user"])
                postobj = classes.Post(
                    api_response={
                        'title': post_data.title,
                        'body': post_data.body,
                        'owner': {
                            'display_name': post_data.owner_name,
                            'reputation': post_data.owner_rep,
                            'link': post_data.owner_url
                        },
                        'site': post_data.site,
                        'IsAnswer': (post_data.post_type == "answer"),
                        'score': post_data.score,
                        'link': post_data.post_url,
                        'question_id': post_data.post_id,
                        'up_vote_count': post_data.up_vote_count,
                        'down_vote_count': post_data.down_vote_count
                    })
                spamhandling.handle_spam(
                    post=postobj,
                    reasons=["Manually reported " + post_data.post_type],
                    why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"][
                            "message"]:
                        if only_blacklists_changed(
                                GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob(
                                    'blacklisted_*.txt'
                            ):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append(
                                                "DOS line ending at `{0}:{1}` in {2}"
                                                .format(
                                                    bl_file, lineno,
                                                    commit_md))
                                        if not line.endswith('\n'):
                                            i.append(
                                                "No newline at end of `{0}` in {1}"
                                                .format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append(
                                                "Blank line at `{0}:{1}` in {2}"
                                                .format(
                                                    bl_file, lineno,
                                                    commit_md))
                                        if line in seen:
                                            i.append(
                                                "Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                .format(
                                                    line.rstrip('\n'),
                                                    seen[line], lineno,
                                                    bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                GlobalVars.charcoal_hq.send_message(
                                    "No code modified in {0}, only blacklists"
                                    " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                GlobalVars.charcoal_hq.send_message(
                                    ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen(
                        'git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            GlobalVars.charcoal_hq.send_message(s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                    # noinspection PyUnboundLocalVariable
                    GlobalVars.charcoal_hq.send_message(s)

            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)
예제 #20
0
    def init_websocket(self):
        try:
            GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host)
            GlobalVars.metasmoke_ws.send(json.dumps({"command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\",\"key\":\"" + GlobalVars.metasmoke_key + "\"}"}))

            while True:
                a = GlobalVars.metasmoke_ws.recv()
                print(a)
                try:
                    data = json.loads(a)
                    if "message" in data:
                        message = data['message']
                        if isinstance(message, Iterable):
                            if "message" in message:
                                GlobalVars.charcoal_hq.send_message(message['message'])
                            elif "blacklist" in message:
                                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']), "metasmoke", message['blacklist']['post'])
                            elif "naa" in message:
                                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                                datahandling.add_ignored_post(post_site_id[0:2])
                            elif "fp" in message:
                                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                                datahandling.add_false_positive(post_site_id[0:2])
                            elif "report" in message:
                                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                                if post_data is None or post_data is False:
                                    continue
                                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                                    continue
                                user = parsing.get_user_from_url(post_data.owner_url)
                                if user is not None:
                                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                                spamhandling.handle_spam(title=post_data.title,
                                                         body=post_data.body,
                                                         poster=post_data.owner_name,
                                                         site=post_data.site,
                                                         post_url=post_data.post_url,
                                                         poster_url=post_data.owner_url,
                                                         post_id=post_data.post_id,
                                                         reasons=["Manually reported " + post_data.post_type],
                                                         is_answer=post_data.post_type == "answer",
                                                         why=why,
                                                         owner_rep=post_data.owner_rep,
                                                         post_score=post_data.score,
                                                         up_vote_count=post_data.up_vote_count,
                                                         down_vote_count=post_data.down_vote_count,
                                                         question_id=post_data.question_id)
                            elif "commit_status" in message:
                                c = message["commit_status"]
                                sha = c["commit_sha"][:7]
                                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                                    if c["status"] == "success":
                                        if "autopull" in c["commit_message"]:
                                            GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"], commit_sha=sha))
                                            time.sleep(2)
                                            os._exit(3)
                                        else:
                                            GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} succeeded.".format(ci_link=c["ci_url"], commit_sha=sha))
                                    elif c["status"] == "failure":
                                        GlobalVars.charcoal_hq.send_message("[CI]({ci_link}) on {commit_sha} failed.".format(ci_link=c["ci_url"], commit_sha=sha))
                except Exception, e:
                    GlobalVars.metasmoke_ws = websocket.create_connection(GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host)
                    GlobalVars.metasmoke_ws.send(json.dumps({"command": "subscribe", "identifier": "{\"channel\":\"SmokeDetectorChannel\"}"}))
                    print e
                    try:
                        exc_info = sys.exc_info()
                        traceback.print_exception(*exc_info)
                    except:
                        print "meh"
        except:
            print "Couldn't bind to MS websocket"
예제 #21
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                GlobalVars.charcoal_hq.send_message(message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
                                                     'owner': {'display_name': post_data.owner_name,
                                                               'reputation': post_data.owner_rep,
                                                               'link': post_data.owner_url},
                                                     'site': post_data.site,
                                                     'IsAnswer': (post_data.post_type == "answer"),
                                                     'score': post_data.score, 'link': post_data.post_url,
                                                     'question_id': post_data.post_id,
                                                     'up_vote_count': post_data.up_vote_count,
                                                     'down_vote_count': post_data.down_vote_count})
                spamhandling.handle_spam(post=postobj,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            GlobalVars.charcoal_hq.send_message(s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                    # noinspection PyUnboundLocalVariable
                    GlobalVars.charcoal_hq.send_message(s)
예제 #22
0
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"],), notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
                                      True, "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                            " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return

            if c["status"] == "success":
                if "autopull" in c["commit_message"]:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
                        os._exit(3)
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                os._exit(6)
def api_get_post(post_url):
    GlobalVars.api_request_lock.acquire()

    # Respect backoff, if we were given one
    if GlobalVars.api_backoff_time > time.time():
        time.sleep(GlobalVars.api_backoff_time - time.time() + 2)
    d = parsing.fetch_post_id_and_site_from_url(post_url)
    if d is None:
        GlobalVars.api_request_lock.release()
        return None
    post_id, site, post_type = d
    if post_type == "answer":
        api_filter = r"!4z6S)cPO)zvpuDWsWTAUW(kaV6K6thsqi1tlYa"
    elif post_type == "question":
        api_filter = r"!m)9.UaQrI5-DZXtlTpWhv2HroYRgS3dPhv.2vxV7fpGT*27rEHM.BKV1"
    else:
        raise ValueError("Unknown post type: {}".format(post_type))

    request_url = "https://api.stackexchange.com/2.2/{}s/{}".format(
        post_type, post_id)
    params = {
        'filter': api_filter,
        'key': 'IAkbitmze4B8KpacUfLqkw((',
        'site': site
    }
    response = requests.get(request_url, params=params).json()
    if "backoff" in response:
        if GlobalVars.api_backoff_time < time.time() + response["backoff"]:
            GlobalVars.api_backoff_time = time.time() + response["backoff"]
    if 'items' not in response or len(response['items']) == 0:
        GlobalVars.api_request_lock.release()
        return False
    GlobalVars.api_request_lock.release()

    item = response['items'][0]
    post_data = PostData()
    post_data.post_id = post_id
    post_data.post_url = parsing.url_to_shortlink(item['link'])
    post_data.post_type = post_type
    post_data.title = html.unescape(item['title'])
    if 'owner' in item and 'link' in item['owner']:
        post_data.owner_name = html.unescape(item['owner']['display_name'])
        post_data.owner_url = item['owner']['link']
        post_data.owner_rep = item['owner']['reputation']
    else:
        post_data.owner_name = ""
        post_data.owner_url = ""
        post_data.owner_rep = 1
    post_data.site = site
    post_data.body = item['body']
    post_data.body_markdown = item['body_markdown']
    post_data.score = item['score']
    post_data.up_vote_count = item['up_vote_count']
    post_data.down_vote_count = item['down_vote_count']
    post_data.creation_date = item['creation_date']
    try:
        post_data.last_edit_date = item['last_edit_date']
    except KeyError:
        post_data.last_edit_date = post_data.creation_date  # Key not present = not edited
    if post_type == "answer":
        post_data.question_id = item['question_id']
    return post_data
예제 #24
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                GlobalVars.charcoal_hq.send_message(message['message'])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                spamhandling.handle_spam(title=post_data.title,
                                         body=post_data.body,
                                         poster=post_data.owner_name,
                                         site=post_data.site,
                                         post_url=post_data.post_url,
                                         poster_url=post_data.owner_url,
                                         post_id=post_data.post_id,
                                         reasons=["Manually reported " + post_data.post_type],
                                         is_answer=post_data.post_type == "answer",
                                         why=why,
                                         owner_rep=post_data.owner_rep,
                                         post_score=post_data.score,
                                         up_vote_count=post_data.up_vote_count,
                                         down_vote_count=post_data.down_vote_count,
                                         question_id=post_data.question_id)
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            GlobalVars.charcoal_hq.send_message(s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                    # noinspection PyUnboundLocalVariable
                    GlobalVars.charcoal_hq.send_message(s)
예제 #25
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("metasmoke", message['message'])
            elif "autoflag_fp" in message:
                event = message["autoflag_fp"]

                chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                           ("no-site-" + event["site"],), notify_site="/autoflag_fp")
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                ids = (message['blacklist']['uid'], message['blacklist']['site'])

                datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
                datahandling.last_feedbacked = (ids, time.time() + 60)
            elif "unblacklist" in message:
                ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
                datahandling.remove_blacklisted_user(ids)
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                post = classes.Post(api_response=post_data.as_dict)

                scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(post)
                if scan_spam:
                    why_append = u"This post would have also been caught for: " + \
                        u", ".join(scan_reasons).capitalize() + "\n" + scan_why
                else:
                    why_append = u"This post would not have been caught otherwise."

                # Add user to blacklist *after* post is scanned
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)

                why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
                    message["report"]["user"], why_append)

                spamhandling.handle_spam(post=post,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log -1 --pretty="%H"').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                                  commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                             commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log -1 --pretty="%H"').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)
예제 #26
0
                def callback(room=room, msg=msg_pings):
                    post = fetch_post_id_and_site_from_url(report_data[0])[0:2]

                    if not datahandling.is_false_positive(post) and not datahandling.is_ignored_post(post):
                        _msg_queue.put((room, msg, report_data))
예제 #27
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data["message"]
        if isinstance(message, Iterable):
            if "message" in message:
                GlobalVars.charcoal_hq.send_message(message["message"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user(
                    (message["blacklist"]["uid"], message["blacklist"]["site"]),
                    "metasmoke",
                    message["blacklist"]["post"],
                )
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(
                    post_data.site, post_data.post_id, post_data.title
                ) and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                spamhandling.handle_spam(
                    title=post_data.title,
                    body=post_data.body,
                    poster=post_data.owner_name,
                    site=post_data.site,
                    post_url=post_data.post_url,
                    poster_url=post_data.owner_url,
                    post_id=post_data.post_id,
                    reasons=["Manually reported " + post_data.post_type],
                    is_answer=post_data.post_type == "answer",
                    why=why,
                    owner_rep=post_data.owner_rep,
                    post_score=post_data.score,
                    up_vote_count=post_data.up_vote_count,
                    down_vote_count=post_data.down_vote_count,
                    question_id=post_data.question_id,
                )
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling...".format(
                                ci_link=c["ci_url"], commit_sha=sha
                            )
                            GlobalVars.charcoal_hq.send_message(s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on {commit_sha} succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on {commit_sha} failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                    # noinspection PyUnboundLocalVariable
                    GlobalVars.charcoal_hq.send_message(s)
예제 #28
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("debug", message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
                                                     'owner': {'display_name': post_data.owner_name,
                                                               'reputation': post_data.owner_rep,
                                                               'link': post_data.owner_url},
                                                     'site': post_data.site,
                                                     'is_answer': (post_data.post_type == "answer"),
                                                     'score': post_data.score, 'link': post_data.post_url,
                                                     'question_id': post_data.post_id,
                                                     'up_vote_count': post_data.up_vote_count,
                                                     'down_vote_count': post_data.down_vote_count})
                spamhandling.handle_spam(post=postobj,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                                  commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                             commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s)
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)
예제 #29
0
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"],), notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
                                      True, "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                            " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return  # Same rev, nothing to do

            if c["status"] == "success":
                if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
                        c["commit_message"].startswith("Auto "):
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
                        exit_mode("pull_update")
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                exit_mode("shutdown")
예제 #30
0
    def init_websocket(self):
        try:
            GlobalVars.metasmoke_ws = websocket.create_connection(
                GlobalVars.metasmoke_ws_host, origin=GlobalVars.metasmoke_host)
            GlobalVars.metasmoke_ws.send(
                json.dumps({
                    "command":
                    "subscribe",
                    "identifier":
                    "{\"channel\":\"SmokeDetectorChannel\",\"key\":\"" +
                    GlobalVars.metasmoke_key + "\"}"
                }))

            while True:
                a = GlobalVars.metasmoke_ws.recv()
                print(a)
                try:
                    data = json.loads(a)
                    if "message" in data:
                        message = data['message']
                        if isinstance(message, Iterable):
                            if "message" in message:
                                GlobalVars.charcoal_hq.send_message(
                                    message['message'])
                            elif "blacklist" in message:
                                datahandling.add_blacklisted_user(
                                    (message['blacklist']['uid'],
                                     message['blacklist']['site']),
                                    "metasmoke", message['blacklist']['post'])
                            elif "naa" in message:
                                post_site_id = parsing.fetch_post_id_and_site_from_url(
                                    message["naa"]["post_link"])
                                datahandling.add_ignored_post(
                                    post_site_id[0:2])
                            elif "fp" in message:
                                post_site_id = parsing.fetch_post_id_and_site_from_url(
                                    message["fp"]["post_link"])
                                datahandling.add_false_positive(
                                    post_site_id[0:2])
                            elif "report" in message:
                                post_data = apigetpost.api_get_post(
                                    message["report"]["post_link"])
                                if post_data is None or post_data is False:
                                    continue
                                if datahandling.has_already_been_posted(
                                        post_data.site, post_data.post_id,
                                        post_data.title
                                ) and not datahandling.is_false_positive(
                                    (post_data.post_id, post_data.site)):
                                    continue
                                user = parsing.get_user_from_url(
                                    post_data.owner_url)
                                if user is not None:
                                    datahandling.add_blacklisted_user(
                                        user, "metasmoke", post_data.post_url)
                                why = u"Post manually reported by user *{}* from metasmoke.\n".format(
                                    message["report"]["user"])
                                spamhandling.handle_spam(
                                    title=post_data.title,
                                    body=post_data.body,
                                    poster=post_data.owner_name,
                                    site=post_data.site,
                                    post_url=post_data.post_url,
                                    poster_url=post_data.owner_url,
                                    post_id=post_data.post_id,
                                    reasons=[
                                        "Manually reported " +
                                        post_data.post_type
                                    ],
                                    is_answer=post_data.post_type == "answer",
                                    why=why,
                                    owner_rep=post_data.owner_rep,
                                    post_score=post_data.score,
                                    up_vote_count=post_data.up_vote_count,
                                    down_vote_count=post_data.down_vote_count,
                                    question_id=post_data.question_id)
                            elif "commit_status" in message:
                                c = message["commit_status"]
                                sha = c["commit_sha"][:7]
                                if c["commit_sha"] != os.popen(
                                        'git log --pretty=format:"%H" -n 1'
                                ).read():
                                    if c["status"] == "success":
                                        if "autopull" in c["commit_message"]:
                                            GlobalVars.charcoal_hq.send_message(
                                                "[CI]({ci_link}) on {commit_sha} succeeded. Message contains 'autopull', pulling..."
                                                .format(ci_link=c["ci_url"],
                                                        commit_sha=sha))
                                            time.sleep(2)
                                            os._exit(3)
                                        else:
                                            GlobalVars.charcoal_hq.send_message(
                                                "[CI]({ci_link}) on {commit_sha} succeeded."
                                                .format(ci_link=c["ci_url"],
                                                        commit_sha=sha))
                                    elif c["status"] == "failure":
                                        GlobalVars.charcoal_hq.send_message(
                                            "[CI]({ci_link}) on {commit_sha} failed."
                                            .format(ci_link=c["ci_url"],
                                                    commit_sha=sha))
                except Exception, e:
                    GlobalVars.metasmoke_ws = websocket.create_connection(
                        GlobalVars.metasmoke_ws_host,
                        origin=GlobalVars.metasmoke_host)
                    GlobalVars.metasmoke_ws.send(
                        json.dumps({
                            "command":
                            "subscribe",
                            "identifier":
                            "{\"channel\":\"SmokeDetectorChannel\"}"
                        }))
                    print e
                    try:
                        exc_info = sys.exc_info()
                        traceback.print_exception(*exc_info)
                    except:
                        print "meh"
        except:
            print "Couldn't bind to MS websocket"
예제 #31
0
    def handle_websocket_data(data):
        if "message" not in data:
            if "type" in data and data['type'] == "reject_subscription":
                log(
                    'error',
                    "MS WebSocket subscription was rejected. Check your MS key."
                )
                raise ConnectionError("MS WebSocket connection rejected")
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"],
                                       ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"], ),
                                       notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke",
                                              message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'],
                   message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]],
                                      message["report"]["user"], True,
                                      "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"][
                        "message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with(
                                "debug",
                                "No code modified in {0}, only blacklists"
                                " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with(
                                "debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            recent_commits = sp.check_output(
                ["git", "log", "-50",
                 "--pretty=%H"]).decode('utf-8').strip().split('\n')
            if c["commit_sha"] in recent_commits:
                return  # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do

            if c["status"] == "success":
                if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
                        c["commit_message"].startswith("Auto "):
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart_blacklists)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart_findspam)
                    else:
                        chatcommunicate.tell_rooms_with('debug',
                                                        s,
                                                        notify_site="/ci")
                        exit_mode("pull_update")
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug",
                                                    s,
                                                    notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                exit_mode("shutdown")
        elif "domain_whitelist" in message:
            if message["domain_whitelist"] == "refresh":
                metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
예제 #32
0
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"],
                                       ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"], ),
                                       notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke",
                                              message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'],
                   message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]],
                                      "the metasmoke API", None,
                                      "the metasmoke API")
            return
            post_data = apigetpost.api_get_post(message["report"]["post_link"])
            if post_data is None or post_data is False:
                return
            if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                    and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                return
            user = parsing.get_user_from_url(post_data.owner_url)
            post = classes.Post(api_response=post_data.as_dict)

            scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(
                post)
            if scan_spam:
                why_append = u"This post would have also been caught for: " + \
                    u", ".join(scan_reasons).capitalize() + "\n" + scan_why
            else:
                why_append = u"This post would not have been caught otherwise."

            # Add user to blacklist *after* post is scanned
            if user is not None:
                datahandling.add_blacklisted_user(user, "metasmoke",
                                                  post_data.post_url)

            why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
                message["report"]["user"], why_append)

            spamhandling.handle_spam(
                post=post,
                reasons=["Manually reported " + post_data.post_type],
                why=why)
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"][
                        "message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with(
                                "debug",
                                "No code modified in {0}, only blacklists"
                                " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with(
                                "debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(
                ["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return

            if c["status"] == "success":
                if "autopull" in c["commit_message"]:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug',
                                                        s,
                                                        notify_site="/ci")
                        os._exit(3)
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug",
                                                    s,
                                                    notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                os._exit(6)