def prepare_git_for_operation(blacklist_file_name): try: git.checkout('master') git.remote.update() git.reset('--hard', 'origin/master') except GitError as e: if GlobalVars.on_windows: return False, "Not doing this, we're on Windows." log_exception(*sys.exc_info()) return False, "`git pull` has failed. This shouldn't happen. Details have been logged." if GlobalVars.on_windows: remote_ref = git.rev_parse("refs/remotes/origin/master").strip() local_ref = git.rev_parse("master").strip() else: remote_ref = git("rev-parse", "refs/remotes/origin/master").strip() local_ref = git("rev-parse", "master").strip() if local_ref != remote_ref: local_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(local_ref)).strip() remote_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(remote_ref)).strip() return False, "HEAD isn't at tip of origin's master branch (local {}, remote {})".format( local_log, remote_log) return True, None
def sync_remote(): try: git.fetch('--force') git.checkout('master', '--force') git.branch('--create-reflog', '-f', 'deploy', '-t', 'origin/deploy') git.checkout('deploy', '--force') git.branch('--create-reflog', '-f', 'master', '-t', 'origin/master') return True, "Synced to origin/master and origin/deploy. You'll probably want to !!/reboot now." except Exception as e: return False, str(e)
def merge_pull_request(cls, pr_id, comment=""): response = requests.get( "https://api.github.com/repos/{}/pulls/{}".format( GlobalVars.bot_repo_slug, pr_id)) if not response: raise ConnectionError("Cannot connect to GitHub API") pr_info = response.json() if pr_info["user"]["login"] != "SmokeDetector": raise ValueError( "PR #{} is not created by me, so I can't approve it.".format( pr_id)) if "<!-- METASMOKE-BLACKLIST" not in pr_info["body"]: raise ValueError("PR description is malformed. Blame a developer.") if pr_info["state"] != "open": raise ValueError( "PR #{} is not currently open, so I won't merge it.".format( pr_id)) ref = pr_info['head']['ref'] if comment: # yay we have comments now GitHubManager.comment_on_thread(pr_id, comment) try: # Remote checks passed, good to go here cls.gitmanager_lock.acquire() git.checkout('master') origin_or_auth = cls.get_origin_or_auth() git.fetch(origin_or_auth, '+refs/pull/{}/head'.format(pr_id)) git( "-c", "user.name=" + GlobalVars.git_name, "-c", "user.email=" + GlobalVars.git_email, "merge", 'FETCH_HEAD', '--no-ff', '-m', 'Merge pull request #{} from {}/{}'.format( pr_id, GlobalVars.bot_repo_slug.split("/")[0], ref)) git.push(origin_or_auth, 'master') try: git.push('-d', origin_or_auth, ref) except GitError as e: # TODO: PR merged, but branch deletion has something wrong, generate some text pass return "Merged pull request [#{0}](https://github.com/{1}/pull/{0}).".format( pr_id, GlobalVars.bot_repo_slug) finally: git.checkout('deploy') cls.gitmanager_lock.release()
def sync_remote_hard(): try: git.fetch('--force') git.checkout('master', '--force') git.reset('origin/master', '--hard') git.checkout('deploy', '--force') git.reset('origin/deploy', '--hard') git.checkout('master', '--force') git.checkout('deploy', '--force') return True, "Synced hard to origin/master and origin/deploy." except Exception as e: return False, str(e)
def pull_remote(): # We need to pull both the master and deploy branches here in order to be in sync with GitHub. git.checkout('deploy') git.pull() git.checkout('master') git.pull() git.checkout('deploy')
def remove_from_blacklist(cls, item, username, blacklist_type="", code_privileged=False, metasmoke_down=False): if not code_privileged: if metasmoke_down: return False, "MS is offline, and I can't determine if you are a blacklist manager or not. " \ "If you are a blacklist manager, then wait for MS to be back up before running " \ "this command." else: return False, "Ask a blacklist manager to run that for you. Use `!!/whois blacklister` to find " \ "out who's here." try: cls.gitmanager_lock.acquire() git.checkout("master") if blacklist_type == "watch": blacklists = [ Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS ] list_type = "watchlist" elif blacklist_type == "blacklist": blacklists = [ Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES, Blacklist.NUMBERS ] list_type = "blacklist" else: return False, "`blacklist_type` not set, blame a developer." for blacklist in blacklists: file_name = blacklist[0] manager = Blacklist(blacklist) exists, _line = manager.exists(item) if exists: break if not exists: return False, 'No such item `{}` in {}.'.format( item, list_type) status, message = cls.prepare_git_for_operation(file_name) if not status: return False, message branch = 'auto-un{}-{}'.format(blacklist_type, time.time()) git.checkout('-b', branch) git.reset('HEAD') manager.remove(item) git.add(file_name) git( "-c", "user.name=" + GlobalVars.git_name, "-c", "user.email=" + GlobalVars.git_email, "commit", "--author={} <{}>".format(GlobalVars.git_name, GlobalVars.git_email), '-m', 'Auto un{} of `{}` by {}'.format(blacklist_type, item, username)) git.checkout('master') git.merge(branch) origin_or_auth = cls.get_origin_or_auth() git.push(origin_or_auth, 'master') try: git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass except Exception as e: log('error', '{}: {}'.format(type(e).__name__, e)) log_exception(*sys.exc_info()) return False, 'Git operations failed for unspecified reasons.' finally: git.checkout('deploy') cls.gitmanager_lock.release() # With no exception raised, list_type should be set return True, 'Removed `{}` from {}'.format(item, list_type)
def add_to_blacklist(cls, blacklist='', item_to_blacklist='', username='', chat_profile_link='', code_permissions=False, metasmoke_down=False): if blacklist == "": return (False, 'GitManager: blacklist is not defined. Blame a developer.') if item_to_blacklist == "": return ( False, 'GitManager: item_to_blacklist is not defined. Blame a developer.' ) # item_to_blacklist = item_to_blacklist.replace("\\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username_is_regex=1&username="******"number": blacklist_type = Blacklist.NUMBERS ms_search_option = "&body=" elif blacklist == "watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "watch_number": blacklist_type = Blacklist.WATCHED_NUMBERS ms_search_option = "&body=" else: return ( False, 'GitManager: blacklist is not recognized. Blame a developer.') blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() status, message = cls.prepare_git_for_operation( blacklist_file_name) if not status: return (False, message) now = str(int(time.time())) if blacklist_type in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: op = 'watch' item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format( op, line, blacklist_file_name)) watch_removed = False if blacklist_type not in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: for watcher_type in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: watcher = Blacklist(watcher_type) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) blacklister.add(item_to_blacklist) branch = "auto-blacklist-{0}".format(now) git.checkout("-b", branch) git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt', 'watched_numbers.txt') git( "-c", "user.name=" + GlobalVars.git_name, "-c", "user.email=" + GlobalVars.git_email, "commit", "--author={} <{}>".format(GlobalVars.git_name, GlobalVars.git_email), "-m", "Auto {0} of `{1}` by {2}".format(op, item, username)) origin_or_auth = cls.get_origin_or_auth() if code_permissions: git.checkout("master") git.merge(branch) git.push(origin_or_auth, "master") git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. else: git.push(origin_or_auth, branch) git.checkout("master") if ((GlobalVars.github_username is None or GlobalVars.github_password is None) and (GlobalVars.github_access_token is None)): return (False, "Tell someone to set a GH token") payload = { "title": "{0}: {1} {2}".format(username, op.title(), item), "body": "[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [in text](https://stackexchange.com/search?q=%22{7}%22)" ", [in URLs](https://stackexchange.com/search?q=url%3A%22{7}%22)" ", and [in code](https://stackexchange.com/search?q=code%3A%22{7}%22)" ".\n" "<!-- METASMOKE-BLACKLIST-{8} {4} -->".format( username, chat_profile_link, op, blacklist, # 0 1 2 3 item, ms_search_option, # 4 5 quote_plus(_anchor(item, blacklist_type)), # 6 quote_plus( item.replace("\\W", " ").replace("\\.", ".")), # 7 blacklist.upper()), # 8 "head": branch, "base": "master" } response = GitHubManager.create_pull_request(payload) log('debug', response) try: git.checkout( "deploy" ) # Return to deploy, pending the accept of the PR in Master. git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. url, pr_num = response["html_url"], response["number"] if metasmoke_down: return ( True, "MS is not reachable, so I can't see if you have blacklist manager privileges, but " "I've [created PR#{1} for you]({0}).{2}".format( url, pr_num, GitHubManager.still_using_usernames_nudge)) else: return (True, "You don't have blacklist manager privileges, " "but I've [created PR#{1} for you]({0}).{2}". format( url, pr_num, GitHubManager.still_using_usernames_nudge)) except KeyError: git.checkout("deploy") # Return to deploy try: # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str(response['message']).lower(): # Capture the case when GH credentials are bad or invalid return ( False, "Something is wrong with the GH credentials, tell someone to check them." ) else: # Capture any other invalid response cases. return ( False, "A bad or invalid reply was received from GH, the message was: %s" % response['message']) except Exception as err: log_exception(*sys.exc_info()) return ( False, "Git functions failed for unspecified reasons, details may be in error log." ) finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted `{0}`".format(item)) elif op == 'watch': return (True, "Added `{0}` to watchlist".format(item))
def test_watch(monkeypatch): chatcommunicate.parse_room_config("test/test_rooms.yml") # XXX TODO: expand def wrap_watch(pattern, force=False): cmd = 'watch{0}'.format('-force' if force else '') msg = Fake({ "_client": { "host": "stackexchange.com", "get_user": lambda id: Fake({ "name": "J F", "id": id }) }, "owner": { "name": "ArtOfCode", "id": 121520 }, "room": { "id": 11540, "get_current_user_ids": lambda: [161943] }, # Ouch, this is iffy # Prevent an error from deep inside do_blacklist "content_source": '!!/{0} {1}'.format(cmd, pattern) }) msg.content = msg.content_source msg.room._client = msg._client return chatcommands.watch(pattern, alias_used=cmd, original_msg=msg) # Prevent from attempting to check privileges with Metasmoke monkeypatch.setattr(GlobalVars, "code_privileged_users", [1, 161943]) try: # Invalid regex resp = wrap_watch(r'?') assert "An invalid pattern was provided" in resp # This is one of the perpetually condemned spam domains, blacklisted forever resp = wrap_watch(r'israelbigmarket') assert "That pattern looks like it's already caught" in resp # The phone number here is the first one which used to be in this format in bad_keywords.txt. # It is now in blacklisted_numbers.txt. resp = wrap_watch(r'[a-z_]*(?:1_*)?913[\W_]*608[\W_]*4584[a-z_]*') assert "Mostly non-latin" not in resp assert "Bad phone number in answer" in resp assert "Bad phone number in body" in resp # XXX TODO: figure out how to trigger duplicate entry separately monkeypatch.setattr("chatcommunicate.is_privileged", lambda *args: True) monkeypatch.setattr("gitmanager.GitManager.prepare_git_for_operation", lambda *args: (True, None)) assert wrap_watch("trimfire", True).startswith("Already watched") monkeypatch.setattr("gitmanager.GitManager.add_to_blacklist", lambda *args, **kwargs: (True, "Hahaha")) assert wrap_watch("male enhancement", True) == "Hahaha" finally: git.checkout("master")
exit_info = [s for s in exit_info if s] # Filter empty strings os.remove("exit.txt") except FileNotFoundError: # Assume something wrong has happened exit_info = [] log('Exit information: [{}] {}'.format(ecode, ", ".join(exit_info) or "None")) if 'no_standby' in exit_info and 'standby' in persistent_arguments: persistent_arguments.remove('standby') elif 'standby' in exit_info and 'standby' not in persistent_arguments: persistent_arguments.append('standby') if 'pull_update' in exit_info: log('Pull in new updates') git.checkout('deploy') git.pull() git.checkout('master') git.merge('@{u}') git.checkout('deploy') count = 0 crashcount = 0 elif 'early_exception' in exit_info: count += 1 log('Incremented crash count: {}; sleeping before restart'.format(count)) sleep(5) if crashcount == 2: log('Crash count triggered reverted state')
os.remove("exit.txt") except FileNotFoundError: # Assume something wrong has happened exit_info = [] log('Exit information: [{}] {}'.format(ecode, ", ".join(exit_info) or "None")) if 'no_standby' in exit_info and 'standby' in persistent_arguments: persistent_arguments.remove('standby') elif 'standby' in exit_info and 'standby' not in persistent_arguments: persistent_arguments.append('standby') if 'pull_update' in exit_info: log('Pull in new updates') git.checkout('deploy') git.pull() count = 0 crashcount = 0 elif 'early_exception' in exit_info: count += 1 log('Incremented crash count: {}; sleeping before restart'.format( count)) sleep(5) if crashcount == 2: log('Crash count triggered reverted state') git.checkout('HEAD~1')