def unwatch(cls, item, username, code_privileged=False): if not code_privileged: return ( False, 'Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who\'s here.' ) try: cls.gitmanager_lock.acquire() watchlist = Blacklist.WATCHED_KEYWORDS file_name = watchlist[0] manager = Blacklist(watchlist) exists, _line = manager.exists(item) if not exists: return (False, 'No such item `{}` in watchlist.'.format(item)) status, message = cls.prepare_git_for_operation(file_name) if not status: return (False, message) branch = 'auto-unwatch-{}'.format(str(time.time())) git.checkout('-b', branch) git.reset('HEAD') manager.remove(item) git.add(file_name) git.commit( "--author='SmokeDetector <*****@*****.**>'", '-m', 'Auto unwatch of {} by {} --autopull'.format(item, username)) git.checkout('master') git.merge(branch) git.push('origin', 'master') git.branch('-D', branch) except Exception as e: log('error', '{}: {}'.format(type(e).__name__, str(e))) return (False, 'Git operations failed for unspecified reasons.') finally: git.checkout('deploy') cls.gitmanager_lock.release() return (True, 'Removed {} from watchlist'.format(item))
def prepare_git_for_operation(blacklist_file_name): try: git.checkout('master') git.remote.update() git.reset('--hard', 'origin/master') except GitError as e: if GlobalVars.on_windows: return False, "Not doing this, we're on Windows." log_exception(*sys.exc_info()) return False, "`git pull` has failed. This shouldn't happen. Details have been logged." if GlobalVars.on_windows: remote_ref = git.rev_parse("refs/remotes/origin/master").strip() local_ref = git.rev_parse("master").strip() else: remote_ref = git("rev-parse", "refs/remotes/origin/master").strip() local_ref = git("rev-parse", "master").strip() if local_ref != remote_ref: local_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(local_ref)).strip() remote_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(remote_ref)).strip() return False, "HEAD isn't at tip of origin's master branch (local {}, remote {})".format( local_log, remote_log) return True, None
def add_to_blacklist(cls, **kwargs): blacklist = kwargs.get("blacklist", "") item_to_blacklist = kwargs.get("item_to_blacklist", "") username = kwargs.get("username", "") chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users") code_permissions = kwargs.get("code_permissions", False) # Make sure git credentials are set up if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`") if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return (False, "Tell someone to run `git config user.email \"[email protected]\"`") if blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return (False, "Programming Error - Critical information missing for GitManager: blacklist") if item_to_blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist") item_to_blacklist = item_to_blacklist.replace("\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username="******"watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body=" else: # Just checking all bases, but blacklist_file_name *might* have empty value # if we don't address it here. return (False, "Invalid blacklist type specified, something has broken badly!") blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() git.checkout("master") try: git.pull() except: pass # Check that we're up-to-date with origin (GitHub) git.remote.update() if 'windows' in platform.platform().lower(): if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip(): return (False, "HEAD isn't at tip of origin's master branch") else: if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip(): return (False, "HEAD isn't at tip of origin's master branch") # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast if blacklist_file_name in git.status(): # Also ugly return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name)) # Set up parameters for watch vs blacklist if blacklist_type in [Blacklist.WATCHED_KEYWORDS]: op = 'watch' now = datetime.now().strftime('%s') item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist # Prevent duplicates exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format(op, line, blacklist_file_name)) # Remove from watch if watched watch_removed = False if blacklist_type not in [Blacklist.WATCHED_KEYWORDS]: watcher = Blacklist(Blacklist.WATCHED_KEYWORDS) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) # Add item to file blacklister.add(item_to_blacklist) # Checkout a new branch (for PRs for non-code-privileged people) branch = "auto-blacklist-{0}".format(str(time.time())) git.checkout("-b", branch) # Clear HEAD just in case git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt') git.commit("--author='SmokeDetector <*****@*****.**>'", "-m", u"Auto {0} of {1} by {2} --autopull".format(op, item, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = {"title": u"{0}: {1} {2}".format(username, op.title(), item), "body": u"[{0}]({1}) requests the {2} of the {3} {4}. See the Metasmoke search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [here](https://stackexchange.com/search?q=%22{6}%22).\n" u"<!-- METASMOKE-BLACKLIST-{7} {4} -->".format( username, chat_profile_link, op, blacklist, item, ms_search_option, quote_plus(item.replace("\\W", " ").replace("\\.", ".")), blacklist.upper()), "head": branch, "base": "master"} response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload)) log('debug', response.json()) try: git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master. git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. url = response.json()["html_url"] return (True, "You don't have code privileges, but I've [created PR#{1} for you]({0}).".format( url, url.split('/')[-1])) except KeyError: git.checkout("deploy") # Return to deploy # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str(response.json()['message']).lower(): # Capture the case when GH credentials are bad or invalid return (False, "Something is wrong with the GH credentials, tell someone to check them.") else: # Capture any other invalid response cases. return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message']) except Exception: return (False, "Git functions failed for unspecified reasons.") finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted {0}".format(item)) elif op == 'watch': return (True, "Added {0} to watchlist".format(item))
def add_to_blacklist(cls, blacklist='', item_to_blacklist='', username='', chat_profile_link='', code_permissions=False, metasmoke_down=False): if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return ( False, 'Tell someone to run `git config user.name "SmokeDetector"`') if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return ( False, 'Tell someone to run `git config user.email "*****@*****.**"`' ) if blacklist == "": return (False, 'GitManager: blacklist is not defined. Blame a developer.') if item_to_blacklist == "": return ( False, 'GitManager: item_to_blacklist is not defined. Blame a developer.' ) # item_to_blacklist = item_to_blacklist.replace("\\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username_is_regex=1&username="******"number": blacklist_type = Blacklist.NUMBERS ms_search_option = "&body=" elif blacklist == "watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "watch_number": blacklist_type = Blacklist.WATCHED_NUMBERS ms_search_option = "&body=" else: return ( False, 'GitManager: blacklist is not recognized. Blame a developer.') blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() status, message = cls.prepare_git_for_operation( blacklist_file_name) if not status: return (False, message) now = str(int(time.time())) if blacklist_type in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: op = 'watch' item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format( op, line, blacklist_file_name)) watch_removed = False if blacklist_type not in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: for watcher_type in { Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS }: watcher = Blacklist(watcher_type) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) blacklister.add(item_to_blacklist) branch = "auto-blacklist-{0}".format(now) git.checkout("-b", branch) git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt', 'watched_numbers.txt') git.commit( "--author='SmokeDetector <*****@*****.**>'", "-m", "Auto {0} of `{1}` by {2} --autopull".format( op, item, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = { "title": "{0}: {1} {2}".format(username, op.title(), item), "body": "[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n" "<!-- METASMOKE-BLACKLIST-{8} {4} -->".format( username, chat_profile_link, op, blacklist, # 0 1 2 3 item, ms_search_option, # 4 5 quote_plus(item), # 6 quote_plus( item.replace("\\W", " ").replace("\\.", ".")), # 7 blacklist.upper()), # 8 "head": branch, "base": "master" } response = GitHubManager.create_pull_request(payload) log('debug', response) try: git.checkout( "deploy" ) # Return to deploy, pending the accept of the PR in Master. git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. url, pr_num = response["html_url"], response["number"] if metasmoke_down: return ( True, "MS is not reachable, so I can't see if you have code privileges, but I've " "[created PR#{1} for you]({0}).".format( url, pr_num)) else: return ( True, "You don't have code privileges, but I've [created PR#{1} for you]({0})." .format(url, pr_num)) except KeyError: git.checkout("deploy") # Return to deploy try: # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str(response['message']).lower(): # Capture the case when GH credentials are bad or invalid return ( False, "Something is wrong with the GH credentials, tell someone to check them." ) else: # Capture any other invalid response cases. return ( False, "A bad or invalid reply was received from GH, the message was: %s" % response['message']) except Exception as err: with open('errorLogs.txt', 'a', encoding="utf-8") as f: f.write("{dt} {message}".format( dt=datetime.now().strftime('%Y-%m-%d %H:%M:%s'), message=str(err))) return ( False, "Git functions failed for unspecified reasons, details may be in error log." ) finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted `{0}`".format(item)) elif op == 'watch': return (True, "Added `{0}` to watchlist".format(item))
def reset_head(): if GlobalVars.on_windows: return # No we don't do Windows git.reset("--hard", "HEAD") git.clean("-f")
def remove_from_blacklist(cls, item, username, blacklist_type="", code_privileged=False, metasmoke_down=False): if not code_privileged: if metasmoke_down: return False, "MS is offline, and I can't determine if you are a code admin or not. " \ "If you are a code admin, then wait for MS to be back up before running this command." else: return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here." try: cls.gitmanager_lock.acquire() git.checkout("master") if blacklist_type == "watch": blacklists = [ Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS ] list_type = "watchlist" elif blacklist_type == "blacklist": blacklists = [ Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES, Blacklist.NUMBERS ] list_type = "blacklist" else: return False, "`blacklist_type` not set, blame a developer." for blacklist in blacklists: file_name = blacklist[0] manager = Blacklist(blacklist) exists, _line = manager.exists(item) if exists: break if not exists: return False, 'No such item `{}` in {}.'.format( item, list_type) status, message = cls.prepare_git_for_operation(file_name) if not status: return False, message branch = 'auto-un{}-{}'.format(blacklist_type, time.time()) git.checkout('-b', branch) git.reset('HEAD') manager.remove(item) git.add(file_name) git.commit( "--author='SmokeDetector <*****@*****.**>'", '-m', 'Auto un{} of `{}` by {} --autopull'.format( blacklist_type, item, username)) git.checkout('master') git.merge(branch) git.push('origin', 'master') try: git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass except Exception as e: log('error', '{}: {}'.format(type(e).__name__, e)) return False, 'Git operations failed for unspecified reasons.' finally: git.checkout('deploy') cls.gitmanager_lock.release() # With no exception raised, list_type should be set return True, 'Removed `{}` from {}'.format(item, list_type)
def add_to_blacklist(**kwargs): blacklist = kwargs.get("blacklist", "") item_to_blacklist = kwargs.get("item_to_blacklist", "") username = kwargs.get("username", "") chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users") code_permissions = kwargs.get("code_permissions", False) # Make sure git credentials are set up if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`") if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return (False, "Tell someone to run `git config user.email \"[email protected]\"`") if blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return (False, "Programming Error - Critical information missing for GitManager: blacklist") if item_to_blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist") item_to_blacklist = item_to_blacklist.replace("\s", " ") if blacklist == "website": blacklist_file_name = "blacklisted_websites.txt" ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_file_name = "bad_keywords.txt" ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_file_name = "blacklisted_usernames.txt" ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!") git.checkout("master") try: git.pull() except: pass # Check that we're up-to-date with origin (GitHub) git.remote.update() if 'windows' in platform.platform().lower(): if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip(): return (False, "HEAD isn't at tip of origin's master branch") else: if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip(): return (False, "HEAD isn't at tip of origin's master branch") # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast if blacklist_file_name in git.status(): # Also ugly return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name)) # Prevent duplicates with open(blacklist_file_name, "r") as blacklist_file: for lineno, line in enumerate(blacklist_file, 1): if line.rstrip('\n') == item_to_blacklist: return (False, '{0} already blacklisted on {1} line {2}'.format( item_to_blacklist, blacklist_file_name, lineno)) # Add item to file with open(blacklist_file_name, "a+") as blacklist_file: last_character = blacklist_file.read()[-1:] if last_character not in ["", "\n"]: blacklist_file.write("\n") blacklist_file.write(item_to_blacklist + "\n") # Checkout a new branch (for PRs for non-code-privileged people) branch = "auto-blacklist-{0}".format(str(time.time())) git.checkout("-b", branch) # Clear HEAD just in case git.reset("HEAD") git.add(blacklist_file_name) git.commit("--author='SmokeDetector <*****@*****.**>'", "-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist), "body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n" u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist, item_to_blacklist, ms_search_option, item_to_blacklist.replace(" ", "+"), blacklist.upper()), "head": branch, "base": "master"} response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload)) log('debug', response.json()) try: git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master. git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format( response.json()["html_url"])) except KeyError: git.checkout("deploy") # Return to deploy # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str(response.json()['message']).lower(): # Capture the case when GH credentials are bad or invalid return (False, "Something is wrong with the GH credentials, tell someone to check them.") else: # Capture any other invalid response cases. return (False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message']) git.checkout("deploy") # Return to deploy to await CI. return (True, "Blacklisted {0}".format(item_to_blacklist))
def reset_head(): if 'windows' in platform.platform().lower(): return # No we don't do Windows git.reset("--hard", "HEAD") git.clean("-f")
def remove_from_blacklist(cls, item, username, blacklist_type="", code_privileged=False): if not code_privileged: return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here." try: cls.gitmanager_lock.acquire() if blacklist_type == "watch": blacklists = [Blacklist.WATCHED_KEYWORDS] list_type = "watchlist" elif blacklist_type == "blacklist": blacklists = [ Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES ] list_type = "blacklist" else: return False, "`blacklist_type` not set, blame a developer." for blacklist in blacklists: file_name = blacklist[0] manager = Blacklist(blacklist) exists, _line = manager.exists(item) if exists: break if not exists: return False, 'No such item `{}` in {}.'.format( item, list_type) status, message = cls.prepare_git_for_operation(file_name) if not status: return False, message branch = 'auto-un{}-{}'.format(blacklist_type, time.time()) git.checkout('-b', branch) git.reset('HEAD') manager.remove(item) git.add(file_name) git.commit( "--author='SmokeDetector <*****@*****.**>'", '-m', 'Auto un{} of `{}` by {} --autopull'.format( blacklist_type, item, username)) git.checkout('master') git.merge(branch) git.push('origin', 'master') git.branch('-D', branch) except Exception as e: log('error', '{}: {}'.format(type(e).__name__, e)) return False, 'Git operations failed for unspecified reasons.' finally: git.checkout('deploy') cls.gitmanager_lock.release() # With no exception raised, list_type should be set return True, 'Removed `{}` from {}'.format(item, list_type)
def add_to_blacklist(cls, **kwargs): blacklist = kwargs.get("blacklist", "") item_to_blacklist = kwargs.get("item_to_blacklist", "") username = kwargs.get("username", "") chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users") code_permissions = kwargs.get("code_permissions", False) # Make sure git credentials are set up if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return ( False, "Tell someone to run `git config user.name \"SmokeDetector\"`") if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return ( False, "Tell someone to run `git config user.email \"[email protected]\"`" ) if blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return ( False, "Programming Error - Critical information missing for GitManager: blacklist" ) if item_to_blacklist == "": # If we broke the code, and this isn't assigned, error out before doing anything, but do # so gracefully with a nice error message. return ( False, "Programming Error - Critical information missing for GitManager: item_to_blacklist" ) item_to_blacklist = item_to_blacklist.replace("\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username_is_regex=1&username="******"watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body_is_regex=1&body=" else: # Just checking all bases, but blacklist_file_name *might* have empty value # if we don't address it here. return ( False, "Invalid blacklist type specified, something has broken badly!" ) blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() git.checkout("master") try: git.pull() except: pass # Check that we're up-to-date with origin (GitHub) git.remote.update() if 'windows' in platform.platform().lower(): if git.rev_parse("refs/remotes/origin/master").strip( ) != git.rev_parse("master").strip(): return (False, "HEAD isn't at tip of origin's master branch") else: if git("rev-parse", "refs/remotes/origin/master").strip() != git( "rev-parse", "master").strip(): return (False, "HEAD isn't at tip of origin's master branch") # Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast if blacklist_file_name in git.status(): # Also ugly return (False, "{0} is modified locally. This is probably bad.". format(blacklist_file_name)) # Set up parameters for watch vs blacklist if blacklist_type in [Blacklist.WATCHED_KEYWORDS]: op = 'watch' now = datetime.now().strftime('%s') item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist # Prevent duplicates exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format( op, line, blacklist_file_name)) # Remove from watch if watched watch_removed = False if blacklist_type not in [Blacklist.WATCHED_KEYWORDS]: watcher = Blacklist(Blacklist.WATCHED_KEYWORDS) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) # Add item to file blacklister.add(item_to_blacklist) # Checkout a new branch (for PRs for non-code-privileged people) branch = "auto-blacklist-{0}".format(str(time.time())) git.checkout("-b", branch) # Clear HEAD just in case git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt') git.commit( "--author='SmokeDetector <*****@*****.**>'", "-m", u"Auto {0} of {1} by {2} --autopull".format( op, item, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = { "title": u"{0}: {1} {2}".format(username, op.title(), item), "body": u"[{0}]({1}) requests the {2} of the {3} `{4}`. See the Metasmoke search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n" u"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format( username, chat_profile_link, op, blacklist, item, ms_search_option, quote_plus(item.replace("\\W", "[- ]")), quote_plus( item.replace("\\W", " ").replace("\\.", ".")), blacklist.upper()), "head": branch, "base": "master" } response = requests.post( "https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload)) log('debug', response.json()) try: git.checkout( "deploy" ) # Return to deploy, pending the accept of the PR in Master. git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. url = response.json()["html_url"] return ( True, "You don't have code privileges, but I've [created PR#{1} for you]({0})." .format(url, url.split('/')[-1])) except KeyError: git.checkout("deploy") # Return to deploy # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str( response.json()['message']).lower(): # Capture the case when GH credentials are bad or invalid return ( False, "Something is wrong with the GH credentials, tell someone to check them." ) else: # Capture any other invalid response cases. return ( False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message']) except Exception: return (False, "Git functions failed for unspecified reasons.") finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted {0}".format(item)) elif op == 'watch': return (True, "Added {0} to watchlist".format(item))
def add_to_blacklist(cls, blacklist='', item_to_blacklist='', username='', chat_profile_link='', code_permissions=False, metasmoke_down=False): if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return (False, 'Tell someone to run `git config user.name "SmokeDetector"`') if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return (False, 'Tell someone to run `git config user.email "*****@*****.**"`') if blacklist == "": return (False, 'GitManager: blacklist is not defined. Blame a developer.') if item_to_blacklist == "": return (False, 'GitManager: item_to_blacklist is not defined. Blame a developer.') # item_to_blacklist = item_to_blacklist.replace("\\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username_is_regex=1&username="******"number": blacklist_type = Blacklist.NUMBERS ms_search_option = "&body=" elif blacklist == "watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "watch_number": blacklist_type = Blacklist.WATCHED_NUMBERS ms_search_option = "&body=" else: return (False, 'GitManager: blacklist is not recognized. Blame a developer.') blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() status, message = cls.prepare_git_for_operation(blacklist_file_name) if not status: return (False, message) now = str(int(time.time())) if blacklist_type in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}: op = 'watch' item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format(op, line, blacklist_file_name)) watch_removed = False if blacklist_type not in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}: for watcher_type in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}: watcher = Blacklist(watcher_type) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) blacklister.add(item_to_blacklist) branch = "auto-blacklist-{0}".format(now) git.checkout("-b", branch) git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt', 'watched_numbers.txt') git.commit("--author='SmokeDetector <*****@*****.**>'", "-m", "Auto {0} of `{1}` by {2}".format(op, item, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = {"title": "{0}: {1} {2}".format(username, op.title(), item), "body": "[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n" "<!-- METASMOKE-BLACKLIST-{8} {4} -->".format( username, chat_profile_link, op, blacklist, # 0 1 2 3 item, ms_search_option, # 4 5 quote_plus(item), # 6 quote_plus(item.replace("\\W", " ").replace("\\.", ".")), # 7 blacklist.upper()), # 8 "head": branch, "base": "master"} response = GitHubManager.create_pull_request(payload) log('debug', response) try: git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master. git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it. url, pr_num = response["html_url"], response["number"] if metasmoke_down: return (True, "MS is not reachable, so I can't see if you have code privileges, but I've " "[created PR#{1} for you]({0}).".format( url, pr_num)) else: return (True, "You don't have code privileges, but I've [created PR#{1} for you]({0}).".format( url, pr_num)) except KeyError: git.checkout("deploy") # Return to deploy try: # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str(response['message']).lower(): # Capture the case when GH credentials are bad or invalid return (False, "Something is wrong with the GH credentials, tell someone to check them.") else: # Capture any other invalid response cases. return (False, "A bad or invalid reply was received from GH, the message was: %s" % response['message']) except Exception as err: log_exception(*sys.exc_info()) return (False, "Git functions failed for unspecified reasons, details may be in error log.") finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted `{0}`".format(item)) elif op == 'watch': return (True, "Added `{0}` to watchlist".format(item))
def remove_from_blacklist(cls, item, username, blacklist_type="", code_privileged=False, metasmoke_down=False): if not code_privileged: if metasmoke_down: return False, "MS is offline, and I can't determine if you are a code admin or not. " \ "If you are a code admin, then wait for MS to be back up before running this command." else: return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here." try: cls.gitmanager_lock.acquire() git.checkout("master") if blacklist_type == "watch": blacklists = [Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS] list_type = "watchlist" elif blacklist_type == "blacklist": blacklists = [Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES, Blacklist.NUMBERS] list_type = "blacklist" else: return False, "`blacklist_type` not set, blame a developer." for blacklist in blacklists: file_name = blacklist[0] manager = Blacklist(blacklist) exists, _line = manager.exists(item) if exists: break if not exists: return False, 'No such item `{}` in {}.'.format(item, list_type) status, message = cls.prepare_git_for_operation(file_name) if not status: return False, message branch = 'auto-un{}-{}'.format(blacklist_type, time.time()) git.checkout('-b', branch) git.reset('HEAD') manager.remove(item) git.add(file_name) git.commit("--author='SmokeDetector <*****@*****.**>'", '-m', 'Auto un{} of `{}` by {}'.format(blacklist_type, item, username)) git.checkout('master') git.merge(branch) git.push('origin', 'master') try: git.branch('-D', branch) except GitError: # It's OK if the branch doesn't get deleted, so long as we switch back to # deploy, which we do in the finally block... pass except Exception as e: log('error', '{}: {}'.format(type(e).__name__, e)) log_exception(*sys.exc_info()) return False, 'Git operations failed for unspecified reasons.' finally: git.checkout('deploy') cls.gitmanager_lock.release() # With no exception raised, list_type should be set return True, 'Removed `{}` from {}'.format(item, list_type)
def add_to_blacklist(cls, blacklist='', item_to_blacklist='', username='', chat_profile_link='', code_permissions=False): if git.config("--get", "user.name", _ok_code=[0, 1]) == "": return ( False, 'Tell someone to run `git config user.name "SmokeDetector"`') if git.config("--get", "user.email", _ok_code=[0, 1]) == "": return ( False, 'Tell someone to run `git config user.email "*****@*****.**"`' ) if blacklist == "": return (False, 'GitManager: blacklist is not defined. Blame a developer.') if item_to_blacklist == "": return ( False, 'GitManager: item_to_blacklist is not defined. Blame a developer.' ) item_to_blacklist = item_to_blacklist.replace("\s", " ") if blacklist == "website": blacklist_type = Blacklist.WEBSITES ms_search_option = "&body_is_regex=1&body=" elif blacklist == "keyword": blacklist_type = Blacklist.KEYWORDS ms_search_option = "&body_is_regex=1&body=" elif blacklist == "username": blacklist_type = Blacklist.USERNAMES ms_search_option = "&username_is_regex=1&username="******"watch_keyword": blacklist_type = Blacklist.WATCHED_KEYWORDS ms_search_option = "&body_is_regex=1&body=" else: return ( False, 'GitManager: blacklist is not recognized. Blame a developer.') blacklister = Blacklist(blacklist_type) blacklist_file_name = blacklist_type[0] try: cls.gitmanager_lock.acquire() status, message = cls.prepare_git_for_operation( blacklist_file_name) if not status: return (False, message) if blacklist_type in [Blacklist.WATCHED_KEYWORDS]: op = 'watch' now = datetime.now().strftime('%s') item = item_to_blacklist item_to_blacklist = "\t".join([now, username, item]) else: op = 'blacklist' item = item_to_blacklist exists, line = blacklister.exists(item_to_blacklist) if exists: return (False, 'Already {}ed on line {} of {}'.format( op, line, blacklist_file_name)) watch_removed = False if blacklist_type not in [Blacklist.WATCHED_KEYWORDS]: watcher = Blacklist(Blacklist.WATCHED_KEYWORDS) if watcher.exists(item_to_blacklist): watch_removed = True watcher.remove(item_to_blacklist) blacklister.add(item_to_blacklist) branch = "auto-blacklist-{0}".format(str(time.time())) git.checkout("-b", branch) git.reset("HEAD") git.add(blacklist_file_name) if watch_removed: git.add('watched_keywords.txt') git.commit( "--author='SmokeDetector <*****@*****.**>'", "-m", u"Auto {0} of {1} by {2} --autopull".format( op, item, username)) if code_permissions: git.checkout("master") git.merge(branch) git.push("origin", "master") git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. else: git.push("origin", branch) git.checkout("master") if GlobalVars.github_username is None or GlobalVars.github_password is None: return (False, "Tell someone to set a GH password") payload = { "title": u"{0}: {1} {2}".format(username, op.title(), item), "body": u"[{0}]({1}) requests the {2} of the {3} `{4}`. See the Metasmoke search [here]" "(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the " "Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n" u"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format( username, chat_profile_link, op, blacklist, item, ms_search_option, quote_plus(item.replace("\\W", "[- ]")), quote_plus( item.replace("\\W", " ").replace("\\.", ".")), blacklist.upper()), "head": branch, "base": "master" } response = requests.post( "https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls", auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password), data=json.dumps(payload)) log('debug', response.json()) try: git.checkout( "deploy" ) # Return to deploy, pending the accept of the PR in Master. git.branch( '-D', branch ) # Delete the branch in the local git tree since we're done with it. url = response.json()["html_url"] return ( True, "You don't have code privileges, but I've [created PR#{1} for you]({0})." .format(url, url.split('/')[-1])) except KeyError: git.checkout("deploy") # Return to deploy # Delete the branch in the local git tree, we'll create it again if the # command is run again. This way, we keep things a little more clean in # the local git tree git.branch('-D', branch) # Error capture/checking for any "invalid" GH reply without an 'html_url' item, # which will throw a KeyError. if "bad credentials" in str( response.json()['message']).lower(): # Capture the case when GH credentials are bad or invalid return ( False, "Something is wrong with the GH credentials, tell someone to check them." ) else: # Capture any other invalid response cases. return ( False, "A bad or invalid reply was received from GH, the message was: %s" % response.json()['message']) except Exception: return (False, "Git functions failed for unspecified reasons.") finally: # Always return to `deploy` branch when done with anything. git.checkout("deploy") cls.gitmanager_lock.release() if op == 'blacklist': return (True, "Blacklisted {0}".format(item)) elif op == 'watch': return (True, "Added {0} to watchlist".format(item))