Exemple #1
0
def pull():
    """
    Pull an update from GitHub
    :return: String on failure, None on success
    """
    if only_blacklists_changed(GitManager.get_remote_diff()):
        GitManager.pull_remote()
        load_blacklists()
        return "No code modified, only blacklists reloaded."
    else:
        request = requests.get('https://api.github.com/repos/Charcoal-SE/SmokeDetector/git/refs/heads/deploy')
        latest_sha = request.json()["object"]["sha"]
        request = requests.get(
            'https://api.github.com/repos/Charcoal-SE/SmokeDetector/commits/{commit_code}/statuses'.format(
                commit_code=latest_sha))
        states = []
        for ci_status in request.json():
            state = ci_status["state"]
            states.append(state)
        if "success" in states:
            os._exit(3)
        elif "error" in states or "failure" in states:
            raise CmdException("CI build failed! :( Please check your commit.")
        elif "pending" in states or not states:
            raise CmdException("CI build is still pending, wait until the build has finished and then pull again.")
Exemple #2
0
def do_blacklist(pattern, blacklist_type, msg, force=False):
    """
    Adds a string to the website blacklist and commits/pushes to GitHub
    :param pattern:
    :param blacklist_type:
    :param msg:
    :param force:
    :return: A string
    """

    chat_user_profile_link = "http://chat.{host}/users/{id}".format(host=msg._client.host,
                                                                    id=msg.owner.id)

    # noinspection PyProtectedMember
    try:
        regex.compile(pattern)
    except regex._regex_core.error:
        raise CmdException("An invalid pattern was provided, not blacklisting.")

    if not force:
        reasons = check_blacklist(pattern.replace("\\W", " ").replace("\\.", "."),
                                  blacklist_type == "username",
                                  blacklist_type == "watch_keyword")

        if reasons:
            raise CmdException("That pattern looks like it's already caught by " + format_blacklist_reasons(reasons) +
                               "; append `-force` if you really want to do that.")

    _, result = GitManager.add_to_blacklist(
        blacklist=blacklist_type,
        item_to_blacklist=pattern,
        username=msg.owner.name,
        chat_profile_link=chat_user_profile_link,
        code_permissions=is_code_privileged(msg._client.host, msg.owner.id)
    )

    return result
Exemple #3
0
    def handle_websocket_data(data):
        if "message" not in data:
            if "type" in data and data['type'] == "reject_subscription":
                log(
                    'error',
                    "MS WebSocket subscription was rejected. Check your MS key."
                )
                raise ConnectionError("MS WebSocket connection rejected")
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"],
                                       ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"], ),
                                       notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke",
                                              message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'],
                   message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]],
                                      message["report"]["user"], True,
                                      "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"][
                        "message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with(
                                "debug",
                                "No code modified in {0}, only blacklists"
                                " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with(
                                "debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            recent_commits = sp.check_output(
                ["git", "log", "-50",
                 "--pretty=%H"]).decode('utf-8').strip().split('\n')
            if c["commit_sha"] in recent_commits:
                return  # Same rev, or earlier rev (e.g. when watching things faster than CI completes), nothing to do

            if c["status"] == "success":
                if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
                        c["commit_message"].startswith("Auto "):
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart_blacklists)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart_findspam)
                    else:
                        chatcommunicate.tell_rooms_with('debug',
                                                        s,
                                                        notify_site="/ci")
                        exit_mode("pull_update")
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug",
                                                    s,
                                                    notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                exit_mode("shutdown")
        elif "domain_whitelist" in message:
            if message["domain_whitelist"] == "refresh":
                metasmoke_cache.MetasmokeCache.delete('whitelisted-domains')
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"],), notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
                                      True, "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                            " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return  # Same rev, nothing to do

            if c["status"] == "success":
                if "autopull" in c["commit_message"] or c["commit_message"].startswith("!") or \
                        c["commit_message"].startswith("Auto "):
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_branch:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            exit_mode("checkout_deploy")
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
                        exit_mode("pull_update")
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                exit_mode("shutdown")
Exemple #5
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                GlobalVars.charcoal_hq.send_message(message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user(
                    (message['blacklist']['uid'],
                     message['blacklist']['site']), "metasmoke",
                    message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(
                    message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(
                    message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(
                    message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke",
                                                      post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(
                    message["report"]["user"])
                postobj = classes.Post(
                    api_response={
                        'title': post_data.title,
                        'body': post_data.body,
                        'owner': {
                            'display_name': post_data.owner_name,
                            'reputation': post_data.owner_rep,
                            'link': post_data.owner_url
                        },
                        'site': post_data.site,
                        'IsAnswer': (post_data.post_type == "answer"),
                        'score': post_data.score,
                        'link': post_data.post_url,
                        'question_id': post_data.post_id,
                        'up_vote_count': post_data.up_vote_count,
                        'down_vote_count': post_data.down_vote_count
                    })
                spamhandling.handle_spam(
                    post=postobj,
                    reasons=["Manually reported " + post_data.post_type],
                    why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"][
                            "message"]:
                        if only_blacklists_changed(
                                GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob(
                                    'blacklisted_*.txt'
                            ):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append(
                                                "DOS line ending at `{0}:{1}` in {2}"
                                                .format(
                                                    bl_file, lineno,
                                                    commit_md))
                                        if not line.endswith('\n'):
                                            i.append(
                                                "No newline at end of `{0}` in {1}"
                                                .format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append(
                                                "Blank line at `{0}:{1}` in {2}"
                                                .format(
                                                    bl_file, lineno,
                                                    commit_md))
                                        if line in seen:
                                            i.append(
                                                "Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                .format(
                                                    line.rstrip('\n'),
                                                    seen[line], lineno,
                                                    bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                GlobalVars.charcoal_hq.send_message(
                                    "No code modified in {0}, only blacklists"
                                    " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                GlobalVars.charcoal_hq.send_message(
                                    ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen(
                        'git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            GlobalVars.charcoal_hq.send_message(s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                    # noinspection PyUnboundLocalVariable
                    GlobalVars.charcoal_hq.send_message(s)

            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)
Exemple #6
0
def remotediff():
    will_require_full_restart = "SmokeDetector will require a full restart to pull changes: " \
                                "{}".format(str(not only_blacklists_changed(GitManager.get_remote_diff())))

    return "{}\n\n{}".format(GitManager.get_remote_diff(), will_require_full_restart)
Exemple #7
0
def gitstatus():
    return GitManager.current_git_status()
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"],), notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]], message["report"]["user"],
                                      True, "the metasmoke API")
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                            " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with("debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return

            if c["status"] == "success":
                if "autopull" in c["commit_message"]:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning', "Pulling remote with HEAD detached, checkout deploy", f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with('debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug', s, notify_site="/ci")
                        os._exit(3)
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                os._exit(6)
Exemple #9
0
    def handle_websocket_data(data):
        if "message" not in data:
            return
        message = data['message']
        if not isinstance(message, Iterable):
            return

        if "message" in message:
            chatcommunicate.tell_rooms_with("metasmoke", message['message'])
        elif "autoflag_fp" in message:
            event = message["autoflag_fp"]

            chatcommunicate.tell_rooms(event["message"],
                                       ("debug", "site-" + event["site"]),
                                       ("no-site-" + event["site"], ),
                                       notify_site="/autoflag_fp")
        elif "exit" in message:
            os._exit(message["exit"])
        elif "blacklist" in message:
            ids = (message['blacklist']['uid'], message['blacklist']['site'])

            datahandling.add_blacklisted_user(ids, "metasmoke",
                                              message['blacklist']['post'])
            datahandling.last_feedbacked = (ids, time.time() + 60)
        elif "unblacklist" in message:
            ids = (message['unblacklist']['uid'],
                   message['unblacklist']['site'])
            datahandling.remove_blacklisted_user(ids)
        elif "naa" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["naa"]["post_link"])
            datahandling.add_ignored_post(post_site_id[0:2])
        elif "fp" in message:
            post_site_id = parsing.fetch_post_id_and_site_from_url(
                message["fp"]["post_link"])
            datahandling.add_false_positive(post_site_id[0:2])
        elif "report" in message:
            import chatcommands  # Do it here
            chatcommands.report_posts([message["report"]["post_link"]],
                                      "the metasmoke API", None,
                                      "the metasmoke API")
            return
            post_data = apigetpost.api_get_post(message["report"]["post_link"])
            if post_data is None or post_data is False:
                return
            if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                    and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                return
            user = parsing.get_user_from_url(post_data.owner_url)
            post = classes.Post(api_response=post_data.as_dict)

            scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(
                post)
            if scan_spam:
                why_append = u"This post would have also been caught for: " + \
                    u", ".join(scan_reasons).capitalize() + "\n" + scan_why
            else:
                why_append = u"This post would not have been caught otherwise."

            # Add user to blacklist *after* post is scanned
            if user is not None:
                datahandling.add_blacklisted_user(user, "metasmoke",
                                                  post_data.post_url)

            why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
                message["report"]["user"], why_append)

            spamhandling.handle_spam(
                post=post,
                reasons=["Manually reported " + post_data.post_type],
                why=why)
        elif "deploy_updated" in message:
            return  # Disabled
            sha = message["deploy_updated"]["head_commit"]["id"]
            if sha != os.popen('git log -1 --pretty="%H"').read():
                if "autopull" in message["deploy_updated"]["head_commit"][
                        "message"]:
                    if only_blacklists_changed(GitManager.get_remote_diff()):
                        commit_md = "[`{0}`](https://github.com/{1}/commit/{0})" \
                                    .format(sha[:7], GlobalVars.bot_repo_slug)
                        integrity = blacklist_integrity_check()
                        if len(integrity) == 0:  # No issues
                            GitManager.pull_remote()
                            findspam.reload_blacklists()
                            chatcommunicate.tell_rooms_with(
                                "debug",
                                "No code modified in {0}, only blacklists"
                                " reloaded.".format(commit_md))
                        else:
                            integrity.append("please fix before pulling.")
                            chatcommunicate.tell_rooms_with(
                                "debug", ", ".join(integrity))
        elif "commit_status" in message:
            c = message["commit_status"]
            sha = c["commit_sha"][:7]
            if c["commit_sha"] == sp.check_output(
                ["git", "log", "-1", "--pretty=%H"]).decode('utf-8').strip():
                return

            if c["status"] == "success":
                if "autopull" in c["commit_message"]:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/" \
                        "commit/{commit_sha}) succeeded. Message contains 'autopull', pulling...".format(
                            ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)
                    remote_diff = GitManager.get_remote_diff()
                    if only_blacklists_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        findspam.FindSpam.reload_blacklists()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart)
                    elif only_modules_changed(remote_diff):
                        GitManager.pull_remote()
                        if not GlobalVars.on_master:
                            # Restart if HEAD detached
                            log('warning',
                                "Pulling remote with HEAD detached, checkout deploy",
                                f=True)
                            os._exit(8)
                        GlobalVars.reload()
                        reload_modules()
                        chatcommunicate.tell_rooms_with(
                            'debug', GlobalVars.s_norestart2)
                    else:
                        chatcommunicate.tell_rooms_with('debug',
                                                        s,
                                                        notify_site="/ci")
                        os._exit(3)
                else:
                    s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                        "succeeded.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                    chatcommunicate.tell_rooms_with("debug",
                                                    s,
                                                    notify_site="/ci")
            elif c["status"] == "failure":
                s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/{repo}/commit/{commit_sha}) " \
                    "failed.".format(ci_link=c["ci_url"], repo=GlobalVars.bot_repo_slug, commit_sha=sha)

                chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
        elif "everything_is_broken" in message:
            if message["everything_is_broken"] is True:
                os._exit(6)
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("debug", message['message'])
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                datahandling.add_blacklisted_user((message['blacklist']['uid'], message['blacklist']['site']),
                                                  "metasmoke", message['blacklist']['post'])
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)
                why = u"Post manually reported by user *{}* from metasmoke.\n".format(message["report"]["user"])
                postobj = classes.Post(api_response={'title': post_data.title, 'body': post_data.body,
                                                     'owner': {'display_name': post_data.owner_name,
                                                               'reputation': post_data.owner_rep,
                                                               'link': post_data.owner_url},
                                                     'site': post_data.site,
                                                     'is_answer': (post_data.post_type == "answer"),
                                                     'score': post_data.score, 'link': post_data.post_url,
                                                     'question_id': post_data.post_id,
                                                     'up_vote_count': post_data.up_vote_count,
                                                     'down_vote_count': post_data.down_vote_count})
                spamhandling.handle_spam(post=postobj,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                                  commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                             commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log --pretty=format:"%H" -n 1').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s)
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s)
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s)
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)
Exemple #11
0
    def handle_websocket_data(data):
        if "message" not in data:
            return

        message = data['message']
        if isinstance(message, Iterable):
            if "message" in message:
                chatcommunicate.tell_rooms_with("metasmoke", message['message'])
            elif "autoflag_fp" in message:
                event = message["autoflag_fp"]

                chatcommunicate.tell_rooms(event["message"], ("debug", "site-" + event["site"]),
                                           ("no-site-" + event["site"],), notify_site="/autoflag_fp")
            elif "exit" in message:
                os._exit(message["exit"])
            elif "blacklist" in message:
                ids = (message['blacklist']['uid'], message['blacklist']['site'])

                datahandling.add_blacklisted_user(ids, "metasmoke", message['blacklist']['post'])
                datahandling.last_feedbacked = (ids, time.time() + 60)
            elif "unblacklist" in message:
                ids = (message['unblacklist']['uid'], message['unblacklist']['site'])
                datahandling.remove_blacklisted_user(ids)
            elif "naa" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["naa"]["post_link"])
                datahandling.add_ignored_post(post_site_id[0:2])
            elif "fp" in message:
                post_site_id = parsing.fetch_post_id_and_site_from_url(message["fp"]["post_link"])
                datahandling.add_false_positive(post_site_id[0:2])
            elif "report" in message:
                post_data = apigetpost.api_get_post(message["report"]["post_link"])
                if post_data is None or post_data is False:
                    return
                if datahandling.has_already_been_posted(post_data.site, post_data.post_id, post_data.title) \
                        and not datahandling.is_false_positive((post_data.post_id, post_data.site)):
                    return
                user = parsing.get_user_from_url(post_data.owner_url)
                post = classes.Post(api_response=post_data.as_dict)

                scan_spam, scan_reasons, scan_why = spamhandling.check_if_spam(post)
                if scan_spam:
                    why_append = u"This post would have also been caught for: " + \
                        u", ".join(scan_reasons).capitalize() + "\n" + scan_why
                else:
                    why_append = u"This post would not have been caught otherwise."

                # Add user to blacklist *after* post is scanned
                if user is not None:
                    datahandling.add_blacklisted_user(user, "metasmoke", post_data.post_url)

                why = u"Post manually reported by user *{}* from metasmoke.\n\n{}".format(
                    message["report"]["user"], why_append)

                spamhandling.handle_spam(post=post,
                                         reasons=["Manually reported " + post_data.post_type],
                                         why=why)
            elif "deploy_updated" in message:
                sha = message["deploy_updated"]["head_commit"]["id"]
                if sha != os.popen('git log -1 --pretty="%H"').read():
                    if "autopull" in message["deploy_updated"]["head_commit"]["message"]:
                        if only_blacklists_changed(GitManager.get_remote_diff()):
                            commit_md = "[`{0}`](https://github.com/Charcoal-SE/SmokeDetector/commit/{0})" \
                                        .format(sha[:7])
                            i = []  # Currently no issues with backlists
                            for bl_file in glob('bad_*.txt') + glob('blacklisted_*.txt'):  # Check blacklists for issues
                                with open(bl_file, 'r') as lines:
                                    seen = dict()
                                    for lineno, line in enumerate(lines, 1):
                                        if line.endswith('\r\n'):
                                            i.append("DOS line ending at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                                  commit_md))
                                        if not line.endswith('\n'):
                                            i.append("No newline at end of `{0}` in {1}".format(bl_file, commit_md))
                                        if line == '\n':
                                            i.append("Blank line at `{0}:{1}` in {2}".format(bl_file, lineno,
                                                                                             commit_md))
                                        if line in seen:
                                            i.append("Duplicate entry of {0} at lines {1} and {2} of {3} in {4}"
                                                     .format(line.rstrip('\n'), seen[line], lineno, bl_file, commit_md))
                                        seen[line] = lineno
                            if i == []:  # No issues
                                GitManager.pull_remote()
                                load_blacklists()
                                chatcommunicate.tell_rooms_with("debug", "No code modified in {0}, only blacklists"
                                                                " reloaded.".format(commit_md))
                            else:
                                i.append("please fix before pulling.")
                                chatcommunicate.tell_rooms_with("debug", ", ".join(i))
            elif "commit_status" in message:
                c = message["commit_status"]
                sha = c["commit_sha"][:7]
                if c["commit_sha"] != os.popen('git log -1 --pretty="%H"').read():
                    if c["status"] == "success":
                        if "autopull" in c["commit_message"]:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha})"\
                                " succeeded. Message contains 'autopull', pulling...".format(ci_link=c["ci_url"],
                                                                                             commit_sha=sha)
                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                            time.sleep(2)
                            os._exit(3)
                        else:
                            s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                                "commit/{commit_sha}) succeeded.".format(ci_link=c["ci_url"], commit_sha=sha)

                            chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
                    elif c["status"] == "failure":
                        s = "[CI]({ci_link}) on [`{commit_sha}`](https://github.com/Charcoal-SE/SmokeDetector/" \
                            "commit/{commit_sha}) failed.".format(ci_link=c["ci_url"], commit_sha=sha)

                        chatcommunicate.tell_rooms_with("debug", s, notify_site="/ci")
            elif "everything_is_broken" in message:
                if message["everything_is_broken"] is True:
                    os._exit(6)