def get_remote_diff():
git.fetch()
if 'windows' in platform.platform().lower():
return git.diff_filenames("deploy", "origin/deploy")
else:
return git("-c", "color.diff=false", "diff", "--name-only",
"deploy", "origin/deploy")
def __init__(self, config=None):
self.log = Log()
self.currentRevision = None
root = None
if config:
if os.path.isdir(config):
root = config
else:
raise Exception(config + ' is not a directory or dont exists!')
else:
root = None
try:
self.git = Git(root)
self.root = self.git.root
try:
self.parse_config()
for target in self.config['targets']:
if target['enabled'] == True:
self.deploy(target)
except Exception as e:
raise e
except Exception as e:
raise e
def pull_local():
diff = GitManager.get_local_diff()
if not only_blacklists_changed(diff):
return
try:
git.merge("--ff-only", "master")
git.push("origin", "deploy")
except GitError:
return
def pull_local():
diff = GitManager.get_local_diff()
if not only_blacklists_changed(diff):
return
try:
git.merge("--ff-only", "master")
git.push("origin", "deploy")
except GitError:
return
def prepare_git_for_operation(blacklist_file_name):
git.checkout('master')
try:
git.pull()
except GitError as e:
if GlobalVars.on_windows:
return False, "Not doing this, we're on Windows."
log_exception(*sys.exc_info())
return False, "`git pull` has failed. This shouldn't happen. Details have been logged."
git.remote.update()
if GlobalVars.on_windows:
remote_ref = git.rev_parse("refs/remotes/origin/master").strip()
local_ref = git.rev_parse("master").strip()
else:
remote_ref = git("rev-parse", "refs/remotes/origin/master").strip()
local_ref = git("rev-parse", "master").strip()
if local_ref != remote_ref:
local_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1",
str(local_ref)).strip()
remote_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1",
str(remote_ref)).strip()
return False, "HEAD isn't at tip of origin's master branch (local {}, remote {})".format(
local_log, remote_log)
if blacklist_file_name in git.status():
return False, "`{}` is modified locally. This is probably bad.".format(
blacklist_file_name)
return True, None
def sync_remote():
if GlobalVars.on_windows:
return False, "We don't do Windows"
try:
if GlobalVars.on_branch != "deploy":
git.checkout("deploy")
git.branch('--create-reflog', '-f', 'master', '-t', 'origin/master')
return True, "wow, this is working!"
except Exception as e:
return False, str(e)
def __init__(self, config = None):
self.log = Log()
self.currentRevision = None
root = None
if config :
if os.path.isdir(config):
root = config
else:
raise Exception(config + ' is not a directory or dont exists!')
else:
root = None
try:
self.git = Git(root);
self.root = self.git.root
try:
self.parse_config()
for target in self.config['targets']:
if target['enabled'] == True:
self.deploy(target)
except Exception as e:
raise e
except Exception as e:
raise e
def __init__(self, current, branch, ssh_path, config):
branch = branch.split('/')[-1]
tmp_path = config['hook']['tmp_path']
#Separate tmp repos per branch
parsed_path = Git.git_url_parse(ssh_path)
tmp = os.path.join(tmp_path, parsed_path['hostname'],
parsed_path['path'], branch)
#tmp is uniqe identifier of repo, this creates front for each repo
if tmp in self.workers:
w = self.workers[tmp]
if w.isAlive():
w.add_work([current, branch, ssh_path, tmp])
else:
self.workers[tmp] = DeployWorker(self)
self.workers[tmp].add_work([current, branch, ssh_path, tmp])
self.workers[tmp].start()
else:
self.workers[tmp] = DeployWorker(self)
self.workers[tmp].add_work([current, branch, ssh_path, tmp])
self.workers[tmp].start()
#clean not running workers
for tmp in self.workers.keys():
if self.workers[tmp].isAlive() == False:
del self.workers[tmp]
def __init__(self, current, branch, ssh_path, config):
branch = branch.split('/')[-1]
tmp_path = config['hook']['tmp_path']
#Separate tmp repos per branch
parsed_path = Git.git_url_parse(ssh_path)
tmp = os.path.join(tmp_path, parsed_path['hostname'], parsed_path['path'], branch)
#tmp is uniqe identifier of repo, this creates front for each repo
if tmp in self.workers:
w = self.workers[tmp]
if w.isAlive():
w.add_work([current, branch, ssh_path, tmp])
else:
self.workers[tmp] = DeployWorker(self)
self.workers[tmp].add_work([current, branch, ssh_path, tmp])
self.workers[tmp].start()
else:
self.workers[tmp] = DeployWorker(self)
self.workers[tmp].add_work([current, branch, ssh_path, tmp])
self.workers[tmp].start()
#clean not running workers
for tmp in self.workers.keys():
if self.workers[tmp].isAlive() == False:
del self.workers[tmp]
def sync_remote():
if GlobalVars.on_windows:
return False, "We don't do Windows"
try:
git('rev-parse', 'temp')
git.branch('-D', 'temp')
except GitError: # local branch 'temp' doesn't exist
pass
git.checkout('-b', 'temp', 'origin/master')
git.branch('-M', 'master')
return True, 'Voilà!'
def prepare_git_for_operation(blacklist_file_name):
git.checkout('master')
try:
git.pull()
except:
pass
git.remote.update()
at_tip = git.rev_parse("refs/remotes/origin/master").strip() == git.rev_parse("master").strip() \
if 'windows' in platform.platform().lower() else \
git("rev-parse", "refs/remotes/origin/master").strip() == git("rev-parse", "master").strip()
if not at_tip:
return (False, "HEAD isn't at tip of origin's master branch")
if blacklist_file_name in git.status():
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
return (True, None)
def prepare_git_for_operation(blacklist_file_name):
try:
git.checkout('master')
git.remote.update()
git.reset('--hard', 'origin/master')
except GitError as e:
if GlobalVars.on_windows:
return False, "Not doing this, we're on Windows."
log_exception(*sys.exc_info())
return False, "`git pull` has failed. This shouldn't happen. Details have been logged."
if GlobalVars.on_windows:
remote_ref = git.rev_parse("refs/remotes/origin/master").strip()
local_ref = git.rev_parse("master").strip()
else:
remote_ref = git("rev-parse", "refs/remotes/origin/master").strip()
local_ref = git("rev-parse", "master").strip()
if local_ref != remote_ref:
local_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(local_ref)).strip()
remote_log = git.log(r"--pretty=`[%h]` *%cn*: %s", "-1", str(remote_ref)).strip()
return False, "HEAD isn't at tip of origin's master branch (local {}, remote {})".format(
local_log, remote_log)
return True, None
def get_local_diff():
if 'windows' in platform.platform().lower():
return git.diff_filenames("HEAD", "master")
else:
return git.diff("--name-only", "HEAD", "master")
def merge_abort():
if GlobalVars.on_windows:
return # No we don't do Windows
git.merge("--abort")
def pull_remote():
git.pull()
def get_remote_diff():
git.fetch()
if GlobalVars.on_windows:
return git.diff_filenames("HEAD", "origin/deploy")
else:
return git.diff("--name-only", "HEAD", "origin/deploy")
def merge_abort():
if GlobalVars.on_windows:
return # No we don't do Windows
git.merge("--abort")
def add_to_blacklist(cls, **kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`")
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username="******"watch_keyword":
blacklist_file_name = "watched_keywords.txt"
ms_search_option = "&body="
else:
# Just checking all bases, but blacklist_file_name *might* have empty value
# if we don't address it here.
return (False, "Invalid blacklist type specified, something has broken badly!")
branch = ""
try:
cls.gitmanager_lock.acquire()
# Fetch remote changes and checkout into detached head
git.fetch("origin", "master")
git.checkout("origin/master")
# Set up parameters for watch vs blacklist
if blacklist_file_name in ['watched_keywords.txt']:
op = 'watch'
now = datetime.now().strftime('%s')
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
item_regex = regex.compile(r'\t\L<item>$', item=[item])
else:
op = 'blacklist'
item = item_to_blacklist
item_regex = regex.compile(r'^\L<item>$', item=[item])
# Prevent duplicates
with open(blacklist_file_name, "r") as blacklist_file:
for lineno, line in enumerate(blacklist_file, 1):
if item_regex.search(line):
return (False, '{0} already {1}ed on {2} line {3}'.format(
item, op, blacklist_file_name, lineno))
# Remove from watch if watched
write_lines = False
if blacklist_file_name not in ['watched_keywords.txt']:
watch_lines = []
watch_regex = regex.compile(r'\t\L<item>$', item=[item])
with open('watched_keywords.txt', 'r') as watch_file:
for lineno, line in enumerate(watch_file, 1):
if watch_regex.search(line):
write_lines = True
continue
watch_lines.append(line)
if write_lines:
with open('watched_keywords.txt', 'w') as watch_file:
for line in watch_lines:
watch_file.write(line)
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character not in ["", "\n"]:
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
git.add(blacklist_file_name)
if write_lines:
git.add('watched_keywords.txt')
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", u"Auto {0} of {1} by {2} --autopull".format(op, item, username))
if code_permissions:
git.push("origin", "HEAD:master")
else:
# Checkout a new branch from detached HEAD
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
git.push("origin", branch)
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: {1} {2}".format(username, op.title(), item),
"body": u"[{0}]({1}) requests the {2} of the {3} {4}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{6}%22).\n"
u"<!-- METASMOKE-BLACKLIST-{7} {4} -->".format(
username, chat_profile_link, op, blacklist,
item, ms_search_option,
quote_plus(item.replace("\\W", " ").replace("\\.", ".")),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/quartata/smokey-git-test/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
url = response.json()["html_url"]
return (True,
"You don't have code privileges, but I've [created PR#{1} for you]({0}).".format(
url, url.split('/')[-1]))
except KeyError:
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
except Exception:
return (False, "Git functions failed for unspecified reasons.")
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
if branch:
git.branch('-D', branch)
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted {0}".format(item))
elif op == 'watch':
return (True, "Added {0} to watchlist".format(item))
def remove_from_blacklist(cls,
item,
username,
blacklist_type="",
code_privileged=False,
metasmoke_down=False):
if not code_privileged:
if metasmoke_down:
return False, "MS is offline, and I can't determine if you are a code admin or not. " \
"If you are a code admin, then wait for MS to be back up before running this command."
else:
return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here."
try:
cls.gitmanager_lock.acquire()
git.checkout("master")
if blacklist_type == "watch":
blacklists = [
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
]
list_type = "watchlist"
elif blacklist_type == "blacklist":
blacklists = [
Blacklist.KEYWORDS, Blacklist.WEBSITES,
Blacklist.USERNAMES, Blacklist.NUMBERS
]
list_type = "blacklist"
else:
return False, "`blacklist_type` not set, blame a developer."
for blacklist in blacklists:
file_name = blacklist[0]
manager = Blacklist(blacklist)
exists, _line = manager.exists(item)
if exists:
break
if not exists:
return False, 'No such item `{}` in {}.'.format(
item, list_type)
status, message = cls.prepare_git_for_operation(file_name)
if not status:
return False, message
branch = 'auto-un{}-{}'.format(blacklist_type, time.time())
git.checkout('-b', branch)
git.reset('HEAD')
manager.remove(item)
git.add(file_name)
git.commit(
"--author='SmokeDetector <*****@*****.**>'", '-m',
'Auto un{} of `{}` by {} --autopull'.format(
blacklist_type, item, username))
git.checkout('master')
git.merge(branch)
git.push('origin', 'master')
try:
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
return False, 'Git operations failed for unspecified reasons.'
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
# With no exception raised, list_type should be set
return True, 'Removed `{}` from {}'.format(item, list_type)
def add_to_blacklist(cls, **kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link",
"http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (
False,
"Tell someone to run `git config user.name \"SmokeDetector\"`")
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (
False,
"Tell someone to run `git config user.email \"[email protected]\"`"
)
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (
False,
"Programming Error - Critical information missing for GitManager: blacklist"
)
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (
False,
"Programming Error - Critical information missing for GitManager: item_to_blacklist"
)
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_type = Blacklist.WEBSITES
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_type = Blacklist.KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_type = Blacklist.USERNAMES
ms_search_option = "&username_is_regex=1&username="******"watch_keyword":
blacklist_type = Blacklist.WATCHED_KEYWORDS
ms_search_option = "&body_is_regex=1&body="
else:
# Just checking all bases, but blacklist_file_name *might* have empty value
# if we don't address it here.
return (
False,
"Invalid blacklist type specified, something has broken badly!"
)
blacklister = Blacklist(blacklist_type)
blacklist_file_name = blacklist_type[0]
try:
cls.gitmanager_lock.acquire()
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if 'windows' in platform.platform().lower():
if git.rev_parse("refs/remotes/origin/master").strip(
) != git.rev_parse("master").strip():
return (False,
"HEAD isn't at tip of origin's master branch")
else:
if git("rev-parse",
"refs/remotes/origin/master").strip() != git(
"rev-parse", "master").strip():
return (False,
"HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False,
"{0} is modified locally. This is probably bad.".
format(blacklist_file_name))
# Set up parameters for watch vs blacklist
if blacklist_type in [Blacklist.WATCHED_KEYWORDS]:
op = 'watch'
now = datetime.now().strftime('%s')
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
else:
op = 'blacklist'
item = item_to_blacklist
# Prevent duplicates
exists, line = blacklister.exists(item_to_blacklist)
if exists:
return (False, 'Already {}ed on line {} of {}'.format(
op, line, blacklist_file_name))
# Remove from watch if watched
watch_removed = False
if blacklist_type not in [Blacklist.WATCHED_KEYWORDS]:
watcher = Blacklist(Blacklist.WATCHED_KEYWORDS)
if watcher.exists(item_to_blacklist):
watch_removed = True
watcher.remove(item_to_blacklist)
# Add item to file
blacklister.add(item_to_blacklist)
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
if watch_removed:
git.add('watched_keywords.txt')
git.commit(
"--author='SmokeDetector <*****@*****.**>'", "-m",
u"Auto {0} of {1} by {2} --autopull".format(
op, item, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {
"title":
u"{0}: {1} {2}".format(username, op.title(), item),
"body":
u"[{0}]({1}) requests the {2} of the {3} `{4}`. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
u"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
username, chat_profile_link, op, blacklist, item,
ms_search_option,
quote_plus(item.replace("\\W", "[- ]")),
quote_plus(
item.replace("\\W", " ").replace("\\.", ".")),
blacklist.upper()),
"head":
branch,
"base":
"master"
}
response = requests.post(
"https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username,
GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
git.checkout(
"deploy"
) # Return to deploy, pending the accept of the PR in Master.
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
url = response.json()["html_url"]
return (
True,
"You don't have code privileges, but I've [created PR#{1} for you]({0})."
.format(url,
url.split('/')[-1]))
except KeyError:
git.checkout("deploy") # Return to deploy
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(
response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (
False,
"Something is wrong with the GH credentials, tell someone to check them."
)
else:
# Capture any other invalid response cases.
return (
False,
"A bad or invalid reply was received from GH, the message was: %s"
% response.json()['message'])
except Exception:
return (False, "Git functions failed for unspecified reasons.")
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted {0}".format(item))
elif op == 'watch':
return (True, "Added {0} to watchlist".format(item))
def add_to_blacklist(cls, blacklist='', item_to_blacklist='', username='', chat_profile_link='',
code_permissions=False, metasmoke_down=False):
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (False, 'Tell someone to run `git config user.name "SmokeDetector"`')
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (False, 'Tell someone to run `git config user.email "*****@*****.**"`')
if blacklist == "":
return (False, 'GitManager: blacklist is not defined. Blame a developer.')
if item_to_blacklist == "":
return (False, 'GitManager: item_to_blacklist is not defined. Blame a developer.')
# item_to_blacklist = item_to_blacklist.replace("\\s", " ")
if blacklist == "website":
blacklist_type = Blacklist.WEBSITES
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_type = Blacklist.KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_type = Blacklist.USERNAMES
ms_search_option = "&username_is_regex=1&username="******"number":
blacklist_type = Blacklist.NUMBERS
ms_search_option = "&body="
elif blacklist == "watch_keyword":
blacklist_type = Blacklist.WATCHED_KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "watch_number":
blacklist_type = Blacklist.WATCHED_NUMBERS
ms_search_option = "&body="
else:
return (False, 'GitManager: blacklist is not recognized. Blame a developer.')
blacklister = Blacklist(blacklist_type)
blacklist_file_name = blacklist_type[0]
try:
cls.gitmanager_lock.acquire()
status, message = cls.prepare_git_for_operation(blacklist_file_name)
if not status:
return (False, message)
now = str(int(time.time()))
if blacklist_type in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}:
op = 'watch'
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
else:
op = 'blacklist'
item = item_to_blacklist
exists, line = blacklister.exists(item_to_blacklist)
if exists:
return (False, 'Already {}ed on line {} of {}'.format(op, line, blacklist_file_name))
watch_removed = False
if blacklist_type not in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}:
for watcher_type in {Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS}:
watcher = Blacklist(watcher_type)
if watcher.exists(item_to_blacklist):
watch_removed = True
watcher.remove(item_to_blacklist)
blacklister.add(item_to_blacklist)
branch = "auto-blacklist-{0}".format(now)
git.checkout("-b", branch)
git.reset("HEAD")
git.add(blacklist_file_name)
if watch_removed:
git.add('watched_keywords.txt', 'watched_numbers.txt')
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", "Auto {0} of `{1}` by {2}".format(op, item, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": "{0}: {1} {2}".format(username, op.title(), item),
"body": "[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
username, chat_profile_link, op, blacklist, # 0 1 2 3
item, ms_search_option, # 4 5
quote_plus(item), # 6
quote_plus(item.replace("\\W", " ").replace("\\.", ".")), # 7
blacklist.upper()), # 8
"head": branch,
"base": "master"}
response = GitHubManager.create_pull_request(payload)
log('debug', response)
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
url, pr_num = response["html_url"], response["number"]
if metasmoke_down:
return (True,
"MS is not reachable, so I can't see if you have code privileges, but I've "
"[created PR#{1} for you]({0}).".format(
url, pr_num))
else:
return (True,
"You don't have code privileges, but I've [created PR#{1} for you]({0}).".format(
url, pr_num))
except KeyError:
git.checkout("deploy") # Return to deploy
try:
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response['message'])
except Exception as err:
log_exception(*sys.exc_info())
return (False, "Git functions failed for unspecified reasons, details may be in error log.")
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted `{0}`".format(item))
elif op == 'watch':
return (True, "Added `{0}` to watchlist".format(item))
def merge_pull_request(cls, pr_id, comment=""):
response = requests.get("https://api.github.com/repos/{}/pulls/{}".format(GlobalVars.bot_repo_slug, pr_id))
if not response:
raise ConnectionError("Cannot connect to GitHub API")
pr_info = response.json()
if pr_info["user"]["login"] != "SmokeDetector":
raise ValueError("PR #{} is not created by me, so I can't approve it".format(pr_id))
if "<!-- METASMOKE-BLACKLIST" not in pr_info["body"]:
raise ValueError("PR description is malformed. Blame a developer")
ref = pr_info['head']['ref']
if comment: # yay we have comments now
GitHubManager.comment_on_thread(pr_id, comment)
try:
# Remote checks passed, good to go here
cls.gitmanager_lock.acquire()
git.checkout('master')
git.fetch('origin', '+refs/pull/{}/head'.format(pr_id))
git.merge('FETCH_HEAD', '--no-ff', '-m', 'Merge pull request #{} from {}/{}'.format(
pr_id, GlobalVars.bot_repo_slug.split("/")[0], ref))
git.push('origin', 'master')
try:
git.push('-d', 'origin', ref)
except GitError as e:
# TODO: PR merged, but branch deletion has something wrong, generate some text
pass
return "Merged pull request [#{0}](https://github.com/{1}/pull/{0}).".format(
pr_id, GlobalVars.bot_repo_slug)
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
def get_local_diff():
if GlobalVars.on_windows:
return git.diff_filenames("HEAD", "master")
else:
return git.diff("--name-only", "HEAD", "master")
def get_remote_diff():
git.fetch()
if GlobalVars.on_windows:
return git.diff_filenames("HEAD", "origin/deploy")
else:
return git.diff("--name-only", "HEAD", "origin/deploy")
def reset_head():
if GlobalVars.on_windows:
return # No we don't do Windows
git.reset("--hard", "HEAD")
git.clean("-f")
def current_git_status():
if 'windows' in platform.platform().lower():
return git.status_stripped()
else:
return git("-c", "color.status=false", "status")
def add_to_blacklist(cls,
blacklist='',
item_to_blacklist='',
username='',
chat_profile_link='',
code_permissions=False,
metasmoke_down=False):
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (
False,
'Tell someone to run `git config user.name "SmokeDetector"`')
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (
False,
'Tell someone to run `git config user.email "*****@*****.**"`'
)
if blacklist == "":
return (False,
'GitManager: blacklist is not defined. Blame a developer.')
if item_to_blacklist == "":
return (
False,
'GitManager: item_to_blacklist is not defined. Blame a developer.'
)
# item_to_blacklist = item_to_blacklist.replace("\\s", " ")
if blacklist == "website":
blacklist_type = Blacklist.WEBSITES
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_type = Blacklist.KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_type = Blacklist.USERNAMES
ms_search_option = "&username_is_regex=1&username="******"number":
blacklist_type = Blacklist.NUMBERS
ms_search_option = "&body="
elif blacklist == "watch_keyword":
blacklist_type = Blacklist.WATCHED_KEYWORDS
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "watch_number":
blacklist_type = Blacklist.WATCHED_NUMBERS
ms_search_option = "&body="
else:
return (
False,
'GitManager: blacklist is not recognized. Blame a developer.')
blacklister = Blacklist(blacklist_type)
blacklist_file_name = blacklist_type[0]
try:
cls.gitmanager_lock.acquire()
status, message = cls.prepare_git_for_operation(
blacklist_file_name)
if not status:
return (False, message)
now = str(int(time.time()))
if blacklist_type in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
op = 'watch'
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
else:
op = 'blacklist'
item = item_to_blacklist
exists, line = blacklister.exists(item_to_blacklist)
if exists:
return (False, 'Already {}ed on line {} of {}'.format(
op, line, blacklist_file_name))
watch_removed = False
if blacklist_type not in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
for watcher_type in {
Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS
}:
watcher = Blacklist(watcher_type)
if watcher.exists(item_to_blacklist):
watch_removed = True
watcher.remove(item_to_blacklist)
blacklister.add(item_to_blacklist)
branch = "auto-blacklist-{0}".format(now)
git.checkout("-b", branch)
git.reset("HEAD")
git.add(blacklist_file_name)
if watch_removed:
git.add('watched_keywords.txt', 'watched_numbers.txt')
git.commit(
"--author='SmokeDetector <*****@*****.**>'", "-m",
"Auto {0} of `{1}` by {2} --autopull".format(
op, item, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {
"title":
"{0}: {1} {2}".format(username, op.title(), item),
"body":
"[{0}]({1}) requests the {2} of the {3} `{4}`. See the MS search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{7}%22).\n"
"<!-- METASMOKE-BLACKLIST-{8} {4} -->".format(
username,
chat_profile_link,
op,
blacklist, # 0 1 2 3
item,
ms_search_option, # 4 5
quote_plus(item), # 6
quote_plus(
item.replace("\\W", " ").replace("\\.", ".")), # 7
blacklist.upper()), # 8
"head":
branch,
"base":
"master"
}
response = GitHubManager.create_pull_request(payload)
log('debug', response)
try:
git.checkout(
"deploy"
) # Return to deploy, pending the accept of the PR in Master.
git.branch(
'-D', branch
) # Delete the branch in the local git tree since we're done with it.
url, pr_num = response["html_url"], response["number"]
if metasmoke_down:
return (
True,
"MS is not reachable, so I can't see if you have code privileges, but I've "
"[created PR#{1} for you]({0}).".format(
url, pr_num))
else:
return (
True,
"You don't have code privileges, but I've [created PR#{1} for you]({0})."
.format(url, pr_num))
except KeyError:
git.checkout("deploy") # Return to deploy
try:
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (
False,
"Something is wrong with the GH credentials, tell someone to check them."
)
else:
# Capture any other invalid response cases.
return (
False,
"A bad or invalid reply was received from GH, the message was: %s"
% response['message'])
except Exception as err:
with open('errorLogs.txt', 'a', encoding="utf-8") as f:
f.write("{dt} {message}".format(
dt=datetime.now().strftime('%Y-%m-%d %H:%M:%s'),
message=str(err)))
return (
False,
"Git functions failed for unspecified reasons, details may be in error log."
)
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted `{0}`".format(item))
elif op == 'watch':
return (True, "Added `{0}` to watchlist".format(item))
def add_to_blacklist(**kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`")
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!")
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if 'windows' in platform.platform().lower():
if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
else:
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Prevent duplicates
with open(blacklist_file_name, "r") as blacklist_file:
for lineno, line in enumerate(blacklist_file, 1):
if line.rstrip('\n') == item_to_blacklist:
return (False, '{0} already blacklisted on {1} line {2}'.format(
item_to_blacklist, blacklist_file_name, lineno))
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character not in ["", "\n"]:
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist),
"body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n"
u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist,
item_to_blacklist, ms_search_option,
item_to_blacklist.replace(" ", "+"),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(
response.json()["html_url"]))
except KeyError:
git.checkout("deploy") # Return to deploy
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
git.checkout("deploy") # Return to deploy to await CI.
return (True, "Blacklisted {0}".format(item_to_blacklist))
class GitDeploy:
root = None
log = None
current_revision = None
git = None
config_file_search = ['deploy.py', 'deploy.ini']
config_file = None
config = {}
revison_file = 'REVISION'
lock_file = 'deploy.lck'
def __init__(self, config=None):
self.log = Log()
self.currentRevision = None
root = None
if config:
if os.path.isdir(config):
root = config
else:
raise Exception(config + ' is not a directory or dont exists!')
else:
root = None
try:
self.git = Git(root)
self.root = self.git.root
try:
self.parse_config()
for target in self.config['targets']:
if target['enabled'] == True:
self.deploy(target)
except Exception as e:
raise e
except Exception as e:
raise e
def parse_config(self):
for conf_file in self.config_file_search:
self.config_file = conf_file
config_file_path = os.path.join(self.root, conf_file)
if os.path.isfile(config_file_path):
break
config = config_reader.configReader(config_file_path)
# this conf file cannot be converted cos it is in repo
#if config_file_path.find('.py') == -1:
# config.migrate_ini2py()
self.config = config.get()
def deploy(self, target_config):
# Parse uri
target_config['uri_parsed'] = urlparse.urlparse(target_config['uri'])
connection = None
if target_config['uri_parsed'].password:
password = target_config['uri_parsed'].password
else:
password = None
if target_config['uri_parsed'].scheme == 'sftp':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 22
connection = Ssh(target_config['uri_parsed'].hostname,
target_config['uri_parsed'].username,
target_config['uri_parsed'].path, port, password)
elif target_config['uri_parsed'].scheme == 'ftp':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 21
connection = Ftp(target_config['uri_parsed'].hostname,
target_config['uri_parsed'].username,
target_config['uri_parsed'].path, port, password)
elif target_config['uri_parsed'].scheme == 'ftps':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 21
connection = Ftps(target_config['uri_parsed'].hostname,
target_config['uri_parsed'].username,
target_config['uri_parsed'].path, port, password)
git_revision = None
git_revision_log = None
try:
if self.current_revision:
git_revision = git_revision_log = self.current_revision
else:
git_revision = git_revision_log = self.git.get_revision()
except Exception as e:
git_revision = None
try:
revision = connection.read_file(
os.path.join(target_config['uri_parsed'].path,
self.revison_file).strip())
except Exception as e:
revision = None
#Revision not match, we must get changes and upload it on server
if git_revision != revision:
#create lock file on remote server
try:
connection.upload_string(
os.path.join(target_config['uri_parsed'].path,
self.lock_file), git_revision_log)
except Exception as err:
self.log.add(str(err), 'error')
if revision and git_revision:
self.log.add(
'Remote revision is {}, current revison is {}'.format(
revision, git_revision), 'ok')
else:
self.log.add(
'No remote revision found, deploying whole project {}'.
format(git_revision_log), 'ok')
files = self.git.diff_commited(revision)
for upload in files['upload']:
if upload.endswith(self.config_file) == False:
try:
premisson = self.check_premisson(upload)
connection.upload_file(
os.path.join(self.root, upload),
os.path.join(target_config['uri_parsed'].path,
upload), premisson)
self.log.add(
'++ Deploying file ' + os.path.join(
target_config['uri_parsed'].path, upload),
'ok')
except Exception as e:
self.log.add(str(e), 'error')
for delete in files['delete']:
try:
connection.delete_file(
os.path.join(target_config['uri_parsed'].path, delete))
self.log.add(
'++ Deleting file ' +
os.path.join(target_config['uri_parsed'].path, delete),
'ok')
except Exception as e:
self.log.add(str(e), 'error')
try:
#destroy lock file
connection.delete_file(
os.path.join(target_config['uri_parsed'].path,
self.lock_file))
except Exception as e:
load.add(str(e), 'error')
try:
#create revision file
connection.upload_string(
os.path.join(target_config['uri_parsed'].path,
self.revison_file), git_revision_log)
except Exception as e:
self.log.add(str(e), 'error')
if target_config['web_hook']:
import urllib2
try:
data = json.dumps({
'errors': len(self.log.get('error')),
'warnings': len(self.log.get('warning'))
})
req = urllib2.Request(target_config['web_hook'], data,
{'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
response = f.read()
f.close()
except:
self.log.add('Calling remote hook failed', 'warning')
self.log.add('Deploy done!', 'ok')
else:
self.log.add('Revisions match, no deploy needed.', 'ok')
def check_premisson(self, filename):
if 'file_rights' in self.config:
for path, premisson in self.config['file_rights'].iteritems():
if filename.endswith(
path
) or path == '*' or '*' in path and filename.startswith(
path.replace('*', '')):
return int(premisson)
return None
def get_log(self):
return self.log
def merge_pull_request(cls, pr_id, comment=""):
response = requests.get(
"https://api.github.com/repos/{}/pulls/{}".format(
GlobalVars.bot_repo_slug, pr_id))
if not response:
raise ConnectionError("Cannot connect to GitHub API")
pr_info = response.json()
if pr_info["user"]["login"] != "SmokeDetector":
raise ValueError(
"PR #{} is not created by me, so I can't approve it".format(
pr_id))
if "<!-- METASMOKE-BLACKLIST" not in pr_info["body"]:
raise ValueError("PR description is malformed. Blame a developer")
ref = pr_info['head']['ref']
if comment: # yay we have comments now
GitHubManager.comment_on_thread(pr_id, comment)
try:
# Remote checks passed, good to go here
cls.gitmanager_lock.acquire()
git.checkout('master')
git.fetch('origin', '+refs/pull/{}/merge'.format(pr_id))
git.merge(
'FETCH_HEAD', '--no-ff', '-m',
'Merge pull request #{} from {}/{} --autopull'.format(
pr_id,
GlobalVars.bot_repo_slug.split("/")[0], ref))
git.push('origin', 'master')
try:
git.push('-d', 'origin', ref)
except GitError as e:
# TODO: PR merged, but branch deletion has something wrong, generate some text
pass
return "Merged pull request [#{0}](https://github.com/{1}/pull/{0}).".format(
pr_id, GlobalVars.bot_repo_slug)
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
def sync(self, tmp, branch, ssh_path): git = Git(tmp) git.update(branch, ssh_path)
def current_git_status():
if GlobalVars.on_windows:
return git.status_stripped()
else:
return str(git.status())
os.remove("exit.txt")
except FileNotFoundError:
# Assume something wrong has happened
exit_info = []
log('Exit information: [{}] {}'.format(ecode, ", ".join(exit_info) or "None"))
if 'no_standby' in exit_info and 'standby' in persistent_arguments:
persistent_arguments.remove('standby')
elif 'standby' in exit_info and 'standby' not in persistent_arguments:
persistent_arguments.append('standby')
if 'pull_update' in exit_info:
log('Pull in new updates')
if not on_windows:
git.checkout('deploy')
git.pull()
else:
warn('Not pulling updates; we are on Windows')
count = 0
crashcount = 0
elif 'early_exception' in exit_info:
count += 1
log('Incremented crash count: {}; sleeping before restart'.format(count))
sleep(5)
if crashcount == 2:
log('Crash count triggered reverted state')
if not on_windows:
def reset_head():
if GlobalVars.on_windows:
return # No we don't do Windows
git.reset("--hard", "HEAD")
git.clean("-f")
def remove_from_blacklist(cls, item, username, blacklist_type="", code_privileged=False, metasmoke_down=False):
if not code_privileged:
if metasmoke_down:
return False, "MS is offline, and I can't determine if you are a code admin or not. " \
"If you are a code admin, then wait for MS to be back up before running this command."
else:
return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here."
try:
cls.gitmanager_lock.acquire()
git.checkout("master")
if blacklist_type == "watch":
blacklists = [Blacklist.WATCHED_KEYWORDS, Blacklist.WATCHED_NUMBERS]
list_type = "watchlist"
elif blacklist_type == "blacklist":
blacklists = [Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES, Blacklist.NUMBERS]
list_type = "blacklist"
else:
return False, "`blacklist_type` not set, blame a developer."
for blacklist in blacklists:
file_name = blacklist[0]
manager = Blacklist(blacklist)
exists, _line = manager.exists(item)
if exists:
break
if not exists:
return False, 'No such item `{}` in {}.'.format(item, list_type)
status, message = cls.prepare_git_for_operation(file_name)
if not status:
return False, message
branch = 'auto-un{}-{}'.format(blacklist_type, time.time())
git.checkout('-b', branch)
git.reset('HEAD')
manager.remove(item)
git.add(file_name)
git.commit("--author='SmokeDetector <*****@*****.**>'",
'-m', 'Auto un{} of `{}` by {}'.format(blacklist_type, item, username))
git.checkout('master')
git.merge(branch)
git.push('origin', 'master')
try:
git.branch('-D', branch)
except GitError:
# It's OK if the branch doesn't get deleted, so long as we switch back to
# deploy, which we do in the finally block...
pass
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
log_exception(*sys.exc_info())
return False, 'Git operations failed for unspecified reasons.'
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
# With no exception raised, list_type should be set
return True, 'Removed `{}` from {}'.format(item, list_type)
def get_local_diff():
if GlobalVars.on_windows:
return git.diff_filenames("HEAD", "master")
else:
return git.diff("--name-only", "HEAD", "master")
def get_remote_diff():
git.fetch()
if 'windows' in platform.platform().lower():
return git.diff_filenames("HEAD", "origin/deploy")
else:
return git.diff("--name-only", "HEAD", "origin/deploy")
def remove_from_blacklist(cls,
item,
username,
blacklist_type="",
code_privileged=False):
if not code_privileged:
return False, "Ask a code admin to run that for you. Use `!!/whois code_admin` to find out who's here."
try:
cls.gitmanager_lock.acquire()
if blacklist_type == "watch":
blacklists = [Blacklist.WATCHED_KEYWORDS]
list_type = "watchlist"
elif blacklist_type == "blacklist":
blacklists = [
Blacklist.KEYWORDS, Blacklist.WEBSITES, Blacklist.USERNAMES
]
list_type = "blacklist"
else:
return False, "`blacklist_type` not set, blame a developer."
for blacklist in blacklists:
file_name = blacklist[0]
manager = Blacklist(blacklist)
exists, _line = manager.exists(item)
if exists:
break
if not exists:
return False, 'No such item `{}` in {}.'.format(
item, list_type)
status, message = cls.prepare_git_for_operation(file_name)
if not status:
return False, message
branch = 'auto-un{}-{}'.format(blacklist_type, time.time())
git.checkout('-b', branch)
git.reset('HEAD')
manager.remove(item)
git.add(file_name)
git.commit(
"--author='SmokeDetector <*****@*****.**>'", '-m',
'Auto un{} of `{}` by {} --autopull'.format(
blacklist_type, item, username))
git.checkout('master')
git.merge(branch)
git.push('origin', 'master')
git.branch('-D', branch)
except Exception as e:
log('error', '{}: {}'.format(type(e).__name__, e))
return False, 'Git operations failed for unspecified reasons.'
finally:
git.checkout('deploy')
cls.gitmanager_lock.release()
# With no exception raised, list_type should be set
return True, 'Removed `{}` from {}'.format(item, list_type)
def current_git_status():
if GlobalVars.on_windows:
return git.status_stripped()
else:
return str(git.status())
persistent_arguments.remove('standby')
except:
pass # We're OK if the argument isn't in the list.
try:
ecode = sp.call(command + persistent_arguments, env=environ)
except KeyboardInterrupt:
# print "[NoCrash] KeyBoard Interrupt received.."
ecode = 6
log('Exited with ecode {}'.format(ecode))
if ecode == 3:
log('Pull in new updates')
if 'windows' not in str(platform.platform()).lower():
git.checkout('deploy')
git.pull()
git.submodule('update')
else:
warn('Not pulling updates; we are on Windows')
count = 0
crashcount = 0
elif ecode == 4:
count += 1
log('Incremented crash count: {}; sleeping before restart'.format(count))
sleep(5)
if crashcount == 2:
log('Crash count triggered reverted state')
def current_git_status():
if 'windows' in platform.platform().lower():
return git.status_stripped()
else:
return git("-c", "color.status=false", "status")
def current_git_status():
if 'windows' in platform.platform().lower():
return git.status_stripped()
else:
return str(git.status())
def pull_remote():
git.pull()
def get_remote_diff():
git.fetch()
if 'windows' in platform.platform().lower():
return git.diff_filenames("deploy", "origin/deploy")
else:
return git("-c", "color.diff=false", "diff", "--name-only", "deploy", "origin/deploy")
def merge_abort():
if 'windows' in platform.platform().lower():
return # No we don't do Windows
git.merge("--abort")
def add_to_blacklist(cls, **kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--get", "user.name", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.name \"SmokeDetector\"`")
if git.config("--get", "user.email", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_type = Blacklist.WEBSITES
ms_search_option = "&body="
elif blacklist == "keyword":
blacklist_type = Blacklist.KEYWORDS
ms_search_option = "&body="
elif blacklist == "username":
blacklist_type = Blacklist.USERNAMES
ms_search_option = "&username="******"watch_keyword":
blacklist_type = Blacklist.WATCHED_KEYWORDS
ms_search_option = "&body="
else:
# Just checking all bases, but blacklist_file_name *might* have empty value
# if we don't address it here.
return (False, "Invalid blacklist type specified, something has broken badly!")
blacklister = Blacklist(blacklist_type)
blacklist_file_name = blacklist_type[0]
try:
cls.gitmanager_lock.acquire()
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if 'windows' in platform.platform().lower():
if git.rev_parse("refs/remotes/origin/master").strip() != git.rev_parse("master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
else:
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Set up parameters for watch vs blacklist
if blacklist_type in [Blacklist.WATCHED_KEYWORDS]:
op = 'watch'
now = datetime.now().strftime('%s')
item = item_to_blacklist
item_to_blacklist = "\t".join([now, username, item])
else:
op = 'blacklist'
item = item_to_blacklist
# Prevent duplicates
exists, line = blacklister.exists(item_to_blacklist)
if exists:
return (False, 'Already {}ed on line {} of {}'.format(op, line, blacklist_file_name))
# Remove from watch if watched
watch_removed = False
if blacklist_type not in [Blacklist.WATCHED_KEYWORDS]:
watcher = Blacklist(Blacklist.WATCHED_KEYWORDS)
if watcher.exists(item_to_blacklist):
watch_removed = True
watcher.remove(item_to_blacklist)
# Add item to file
blacklister.add(item_to_blacklist)
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
if watch_removed:
git.add('watched_keywords.txt')
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", u"Auto {0} of {1} by {2} --autopull".format(op, item, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: {1} {2}".format(username, op.title(), item),
"body": u"[{0}]({1}) requests the {2} of the {3} {4}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{5}{6}) and the "
"Stack Exchange search [here](https://stackexchange.com/search?q=%22{6}%22).\n"
u"<!-- METASMOKE-BLACKLIST-{7} {4} -->".format(
username, chat_profile_link, op, blacklist,
item, ms_search_option,
quote_plus(item.replace("\\W", " ").replace("\\.", ".")),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
url = response.json()["html_url"]
return (True,
"You don't have code privileges, but I've [created PR#{1} for you]({0}).".format(
url, url.split('/')[-1]))
except KeyError:
git.checkout("deploy") # Return to deploy
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
except Exception:
return (False, "Git functions failed for unspecified reasons.")
finally:
# Always return to `deploy` branch when done with anything.
git.checkout("deploy")
cls.gitmanager_lock.release()
if op == 'blacklist':
return (True, "Blacklisted {0}".format(item))
elif op == 'watch':
return (True, "Added {0} to watchlist".format(item))
def reset_head():
if 'windows' in platform.platform().lower():
return # No we don't do Windows
git.reset("--hard", "HEAD")
git.clean("-f")
persistent_arguments.remove('standby')
except:
pass # We're OK if the argument isn't in the list.
try:
ecode = sp.call(command + persistent_arguments, env=environ)
except KeyboardInterrupt:
# print "[NoCrash] KeyBoard Interrupt received.."
ecode = 6
log('Exited with ecode {}'.format(ecode))
if ecode == 3:
log('Pull in new updates')
if 'windows' not in str(platform.platform()).lower():
git.checkout('deploy')
git.pull()
git.submodule('update')
else:
warn('Not pulling updates; we are on Windows')
count = 0
crashcount = 0
elif ecode == 4:
count += 1
log('Incremented crash count: {}; sleeping before restart'.format(
count))
sleep(5)
if crashcount == 2:
class GitDeploy:
root = None
log = None
current_revision = None
git = None
config_file_search = ['deploy.py', 'deploy.ini']
config_file = None
config = {}
revison_file = 'REVISION'
lock_file = 'deploy.lck'
def __init__(self, config = None):
self.log = Log()
self.currentRevision = None
root = None
if config :
if os.path.isdir(config):
root = config
else:
raise Exception(config + ' is not a directory or dont exists!')
else:
root = None
try:
self.git = Git(root);
self.root = self.git.root
try:
self.parse_config()
for target in self.config['targets']:
if target['enabled'] == True:
self.deploy(target)
except Exception as e:
raise e
except Exception as e:
raise e
def parse_config(self):
for conf_file in self.config_file_search:
self.config_file = conf_file
config_file_path = os.path.join(self.root, conf_file)
if os.path.isfile(config_file_path):
break
config = config_reader.configReader(config_file_path)
# this conf file cannot be converted cos it is in repo
#if config_file_path.find('.py') == -1:
# config.migrate_ini2py()
self.config = config.get()
def deploy(self, target_config):
# Parse uri
target_config['uri_parsed'] = urlparse.urlparse(target_config['uri'])
connection = None
if target_config['uri_parsed'].password:
password = target_config['uri_parsed'].password
else:
password = None
if target_config['uri_parsed'].scheme == 'sftp':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 22
connection = Ssh(target_config['uri_parsed'].hostname, target_config['uri_parsed'].username, target_config['uri_parsed'].path, port, password)
elif target_config['uri_parsed'].scheme == 'ftp':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 21
connection = Ftp(target_config['uri_parsed'].hostname, target_config['uri_parsed'].username, target_config['uri_parsed'].path, port, password)
elif target_config['uri_parsed'].scheme == 'ftps':
if target_config['uri_parsed'].port:
port = target_config['uri_parsed'].port
else:
port = 21
connection = Ftps(target_config['uri_parsed'].hostname, target_config['uri_parsed'].username, target_config['uri_parsed'].path, port, password)
git_revision = None;
git_revision_log = None;
try:
if self.current_revision:
git_revision = git_revision_log = self.current_revision
else:
git_revision = git_revision_log = self.git.get_revision()
except Exception as e:
git_revision = None;
try:
revision = connection.read_file(os.path.join(target_config['uri_parsed'].path, self.revison_file).strip())
except Exception as e:
revision = None;
#Revision not match, we must get changes and upload it on server
if git_revision != revision:
#create lock file on remote server
try:
connection.upload_string(os.path.join(target_config['uri_parsed'].path, self.lock_file), git_revision_log)
except Exception as err:
self.log.add(str(err), 'error')
if revision and git_revision:
self.log.add('Remote revision is {}, current revison is {}'.format(revision, git_revision), 'ok')
else:
self.log.add('No remote revision found, deploying whole project {}'.format(git_revision_log), 'ok')
files = self.git.diff_commited(revision);
for upload in files['upload']:
if upload.endswith(self.config_file) == False:
try:
premisson = self.check_premisson(upload)
connection.upload_file(os.path.join(self.root, upload), os.path.join(target_config['uri_parsed'].path, upload), premisson)
self.log.add('++ Deploying file ' + os.path.join(target_config['uri_parsed'].path, upload), 'ok')
except Exception as e:
self.log.add(str(e), 'error')
for delete in files['delete']:
try:
connection.delete_file(os.path.join(target_config['uri_parsed'].path, delete))
self.log.add('++ Deleting file ' + os.path.join(target_config['uri_parsed'].path, delete), 'ok')
except Exception as e:
self.log.add(str(e), 'error')
try:
#destroy lock file
connection.delete_file(os.path.join(target_config['uri_parsed'].path, self.lock_file))
except Exception as e:
load.add(str(e), 'error')
try:
#create revision file
connection.upload_string(os.path.join(target_config['uri_parsed'].path, self.revison_file), git_revision_log)
except Exception as e:
self.log.add(str(e), 'error')
if target_config['web_hook']:
import urllib2
try:
data = json.dumps({'errors': len(self.log.get('error')), 'warnings': len(self.log.get('warning'))})
req = urllib2.Request(target_config['web_hook'], data, {'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
response = f.read()
f.close()
except:
self.log.add('Calling remote hook failed', 'warning')
self.log.add('Deploy done!', 'ok')
else:
self.log.add('Revisions match, no deploy needed.', 'ok')
def check_premisson(self, filename):
if 'file_rights' in self.config:
for path, premisson in self.config['file_rights'].iteritems():
if filename.endswith(path) or path == '*' or '*' in path and filename.startswith(path.replace('*', '')):
return int(premisson)
return None
def get_log(self):
return self.log
