def prepare_git_for_operation(blacklist_file_name):
try:
git.checkout('master')
git.remote.update()
git.reset('--hard', 'origin/master')
except GitError as e:
if GlobalVars.on_windows:
return False, "Not doing this, we're on Windows."
log_exception(*sys.exc_info())
return False, "`git pull` has failed. This shouldn't happen. Details have been logged."
if GlobalVars.on_windows:
remote_ref = git.rev_parse("refs/remotes/origin/master").strip()
local_ref = git.rev_parse("master").strip()
else:
remote_ref = git("rev-parse", "refs/remotes/origin/master").strip()
local_ref = git("rev-parse", "master").strip()
if local_ref != remote_ref:
local_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1",
str(local_ref)).strip()
remote_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1",
str(remote_ref)).strip()
return False, "HEAD isn't at tip of origin's master branch (local {}, remote {})".format(
local_log, remote_log)
return True, None
def test_watch(monkeypatch):
# XXX TODO: expand
def wrap_watch(pattern, force=False):
cmd = 'watch{0}'.format('-force' if force else '')
msg = Fake({
"_client": {
"host": "stackexchange.com",
"get_user": lambda id: Fake({
"name": "J F",
"id": id
})
},
"owner": {
"name": "ArtOfCode",
"id": 121520
},
"room": {
"id": 11540,
"get_current_user_ids": lambda: [161943]
},
# Ouch, this is iffy
# Prevent an error from deep inside do_blacklist
"content_source": '!!/{0} {1}'.format(cmd, pattern)
})
msg.room._client = msg._client
return chatcommands.watch(pattern, alias_used=cmd, original_msg=msg)
# Prevent from attempting to check privileges with Metasmoke
monkeypatch.setattr(GlobalVars, "code_privileged_users", [1, 161943])
try:
# Invalid regex
resp = wrap_watch(r'?')
assert "An invalid pattern was provided" in resp
# This is one of the perpetually condemned spam domains, blacklisted forever
resp = wrap_watch(r'israelbigmarket')
assert "That pattern looks like it's already caught" in resp
# The phone number here is the first one in this format in bad_keywords.txt
resp = wrap_watch(r'[a-z_]*(?:1_*)?913[\W_]*608[\W_]*4584[a-z_]*')
assert "Mostly non-latin" not in resp
assert "Bad keyword in answer" in resp
assert "Bad keyword in body" in resp
# XXX TODO: figure out how to trigger duplicate entry separately
monkeypatch.setattr("chatcommunicate.is_privileged",
lambda *args: True)
monkeypatch.setattr("gitmanager.GitManager.prepare_git_for_operation",
lambda *args: (True, None))
assert wrap_watch("trimfire", True).startswith("Already watched")
monkeypatch.setattr("gitmanager.GitManager.add_to_blacklist",
lambda *args, **kwargs: (True, "Hahaha"))
assert wrap_watch("male enhancement", True) == "Hahaha"
finally:
git.checkout("master")
def sync_remote():
try:
if GlobalVars.on_branch != "deploy":
git.checkout("deploy")
git.branch('--create-reflog', '-f', 'master', '-t',
'origin/master')
return True, "wow, this is working!"
except Exception as e:
return False, str(e)
def sync_remote():
try:
git.fetch('--force')
git.checkout('master', '--force')
git.branch('--create-reflog', '-f', 'deploy', '-t', 'origin/deploy')
git.checkout('deploy', '--force')
git.branch('--create-reflog', '-f', 'master', '-t', 'origin/master')
return True, "Synced to origin/master and origin/deploy. You'll probably want to !!/reboot now."
except Exception as e:
return False, str(e)
def merge_pull_request(cls, pr_id, comment=""):
response = requests.get(
"https://api.github.com/repos/{}/pulls/{}".format(
GlobalVars.bot_repo_slug, pr_id))
if not response:
raise ConnectionError("Cannot connect to GitHub API")
pr_info = response.json()
if pr_info["user"]["login"] != "SmokeDetector":
raise ValueError(
"PR #{} is not created by me, so I can't approve it.".format(
pr_id))
if "<!-- METASMOKE-BLACKLIST" not in pr_info["body"]:
raise ValueError("PR description is malformed. Blame a developer.")
if pr_info["state"] != "open":
raise ValueError(
"PR #{} is not currently open, so I won't merge it.".format(
pr_id))
ref = pr_info['head']['ref']
if comment: # yay we have comments now
GitHubManager.comment_on_thread(pr_id, comment)
try:
# Remote checks passed, good to go here
cls.gitmanager_lock.acquire()
git.checkout('master')
origin_or_auth = cls.get_origin_or_auth()
git.fetch(origin_or_auth, '+refs/pull/{}/head'.format(pr_id))
git(
"-c", "user.name=" + GlobalVars.git_name, "-c",
"user.email=" + GlobalVars.git_email, "merge", 'FETCH_HEAD',
'--no-ff', '-m', 'Merge pull request #{} from {}/{}'.format(
pr_id,
GlobalVars.bot_repo_slug.split("/")[0], ref))
git.push(origin_or_auth, 'master')
try:
git.push('-d', origin_or_auth, ref)
except GitError as e:
# TODO: PR merged, but branch deletion has something wrong, generate some text
pass
return "Merged pull request [#{0}](https://github.com/{1}/pull/{0}).".format(
pr_id, GlobalVars.bot_repo_slug)
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
def sync_remote_hard():
try:
git.fetch('--force')
git.checkout('master', '--force')
git.reset('origin/master', '--hard')
git.checkout('deploy', '--force')
git.reset('origin/deploy', '--hard')
git.checkout('master', '--force')
git.checkout('deploy', '--force')
return True, "Synced hard to origin/master and origin/deploy."
except Exception as e:
return False, str(e)
def add_to_blacklist(cls,
blacklist='',
item_to_blacklist='',
username='',
chat_profile_link='',
code_permissions=False,
metasmoke_down=False):
if blacklist == "":
return (False,
'GitManager: blacklist is not defined. Blame a developer.')
if item_to_blacklist == "":
return (
False,
'GitManager: item_to_blacklist is not defined. Blame a developer.'
)
# item_to_blacklist = item_to_blacklist.replace("\\s", " ")
if blacklist == "website":
blacklist_type = Blacklist.WEBSITES
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_type = Blacklist.KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_type = Blacklist.USERNAMES
ms_search_option = "&username_is_regex=1&username="******"number":
blacklist_type = Blacklist.NUMBERS
ms_search_option = "&body="
elif blacklist == "watch_keyword":
blacklist_type = Blacklist.WATCHED_KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "watch_number":
blacklist_type = Blacklist.WATCHED_NUMBERS
ms_search_option = "&body="
else:
return (
False,
'GitManager: blacklist is not recognized. Blame a developer.')
blacklister = Blacklist(blacklist_type)
blacklist_file_name = blacklist_type[0]
try:
cls.gitmanager_lock.acquire()
status, message = cls.prepare_git_for_operation(
blacklist_file_name)
if not status:
return (False, message)
now = str(int(time.time()))
if blacklist_type in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
op = 'watch'
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
else:
op = 'blacklist'
item = item_to_blacklist
exists, line = blacklister.exists(item_to_blacklist)
if exists:
return (False, 'Already {}ed on line {} of {}'.format(
op, line, blacklist_file_name))
watch_removed = False
if blacklist_type not in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
for watcher_type in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
watcher = Blacklist(watcher_type)
if watcher.exists(item_to_blacklist):
watch_removed = True
watcher.remove(item_to_blacklist)
blacklister.add(item_to_blacklist)
branch = "auto-blacklist-{0}".format(now)
git.checkout("-b", branch)
git.reset("HEAD")
git.add(blacklist_file_name)
if watch_removed:
git.add('watched_keywords.txt', 'watched_numbers.txt')
git(
"-c", "user.name=" + GlobalVars.git_name, "-c",
"user.email=" + GlobalVars.git_email, "commit",
"--author={} <{}>".format(GlobalVars.git_name,
GlobalVars.git_email), "-m",
"Auto {0} of `{1}` by {2}".format(op, item, username))
origin_or_auth = cls.get_origin_or_auth()
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push(origin_or_auth, "master")
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
else:
git.push(origin_or_auth, branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {
"title":
"{0}: {1} {2}".format(username, op.title(), item),
"body":
"[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
username,
chat_profile_link,
op,
blacklist, # 0 1 2 3
item,
ms_search_option, # 4 5
quote_plus(item), # 6
quote_plus(
item.replace("\\W", " ").replace("\\.", ".")), # 7
blacklist.upper()), # 8
"head":
branch,
"base":
"master"
}
response = GitHubManager.create_pull_request(payload)
log('debug', response)
try:
git.checkout(
"deploy"
) # Return to deploy, pending the accept of the PR in Master.
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
url, pr_num = response["html_url"], response["number"]
if metasmoke_down:
return (
True,
"MS is not reachable, so I can't see if you have blacklist manager privileges, but "
"I've [created PR#{1} for you]({0}).".format(
url, pr_num))
else:
return (
True,
"You don't have blacklist manager privileges, but I've [created PR#{1} for you]({0})."
.format(url, pr_num))
except KeyError:
git.checkout("deploy") # Return to deploy
try:
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (
False,
"Something is wrong with the GH credentials, tell someone to check them."
)
else:
# Capture any other invalid response cases.
return (
False,
"A bad or invalid reply was received from GH, the message was: %s"
% response['message'])
except Exception as err:
log_exception(*sys.exc_info())
return (
False,
"Git functions failed for unspecified reasons, details may be in error log."
)
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted `{0}`".format(item))
elif op == 'watch':
return (True, "Added `{0}` to watchlist".format(item))
def remove_from_blacklist(cls,
item,
username,
blacklist_type="",
code_privileged=False,
metasmoke_down=False):
if not code_privileged:
if metasmoke_down:
return False, "MS is offline, and I can't determine if you are a blacklist manager or not. " \
"If you are a blacklist manager, then wait for MS to be back up before running " \
"this command."
else:
return False, "Ask a blacklist manager to run that for you. Use `!!/whois blacklister` to find " \
"out who's here."
try:
cls.gitmanager_lock.acquire()
git.checkout("master")
if blacklist_type == "watch":
blacklists = [
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
]
list_type = "watchlist"
elif blacklist_type == "blacklist":
blacklists = [
Blacklist.KEYWORDS, Blacklist.WEBSITES,
Blacklist.USERNAMES, Blacklist.NUMBERS
]
list_type = "blacklist"
else:
return False, "`blacklist_type` not set, blame a developer."
for blacklist in blacklists:
file_name = blacklist[0]
manager = Blacklist(blacklist)
exists, _line = manager.exists(item)
if exists:
break
if not exists:
return False, 'No such item `{}` in {}.'.format(
item, list_type)
status, message = cls.prepare_git_for_operation(file_name)
if not status:
return False, message
branch = 'auto-un{}-{}'.format(blacklist_type, time.time())
git.checkout('-b', branch)
git.reset('HEAD')
manager.remove(item)
git.add(file_name)
git(
"-c", "user.name=" + GlobalVars.git_name, "-c",
"user.email=" + GlobalVars.git_email, "commit",
"--author={} <{}>".format(GlobalVars.git_name,
GlobalVars.git_email), '-m',
'Auto un{} of `{}` by {}'.format(blacklist_type, item,
username))
git.checkout('master')
git.merge(branch)
origin_or_auth = cls.get_origin_or_auth()
git.push(origin_or_auth, 'master')
try:
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
log_exception(*sys.exc_info())
return False, 'Git operations failed for unspecified reasons.'
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
# With no exception raised, list_type should be set
return True, 'Removed `{}` from {}'.format(item, list_type)
os.remove("exit.txt")
except FileNotFoundError:
# Assume something wrong has happened
exit_info = []
log('Exit information: [{}] {}'.format(ecode, ", ".join(exit_info)
or "None"))
if 'no_standby' in exit_info and 'standby' in persistent_arguments:
persistent_arguments.remove('standby')
elif 'standby' in exit_info and 'standby' not in persistent_arguments:
persistent_arguments.append('standby')
if 'pull_update' in exit_info:
log('Pull in new updates')
git.checkout('deploy')
git.pull()
count = 0
crashcount = 0
elif 'early_exception' in exit_info:
count += 1
log('Incremented crash count: {}; sleeping before restart'.format(
count))
sleep(5)
if crashcount == 2:
log('Crash count triggered reverted state')
git.checkout('HEAD~1')
