def test_init_git_create_repo(self):
git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
shutil.rmtree(git_dir)
Git()
self.assertTrue(os.path.isdir(git_dir))
self.assertEqual(git.config('user.name').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_NAME)
self.assertEqual(git.config('user.email').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_EMAIL)
def run_init(*args, **kwargs):
# initialize the git repository if not already done
try:
git("rev-parse", "--dir-list")
except ErrorReturnCode_128:
git("init")
exclude = kwargs.get('exclude')
hooks_added = []
for hook in SUPPORTED_HOOKS:
if exclude and hook not in exclude or not exclude:
try:
os.symlink(path_to_the_script, '.git/hooks/{}'.format(hook))
except OSError, e:
if e.errno == 17:
logging.info('Already initialized')
# create a temporary hook key/val
key = 'hooks.{hook_type}.placeholder'.format(hook_type=hook)
val = 'true'
try:
git.config('--get', key)
except ErrorReturnCode_1:
git.config('--add', key, val)
hooks_added.append(hook)
def __init__(self):
self.__dict__ = self.__shared_state
from waliki.settings import WALIKI_DATA_DIR
self.content_dir = WALIKI_DATA_DIR
os.chdir(self.content_dir)
if not os.path.isdir(os.path.join(self.content_dir, '.git')):
git.init()
git.config("user.email", settings.WALIKI_COMMITTER_EMAIL)
git.config("user.name", settings.WALIKI_COMMITTER_NAME)
def test_require_git_email_name(self, tmpdir):
k = {'_cwd': str(tmpdir)}
git('init', **k)
with pytest.raises(Exception) as excinfo:
i18n_tool.I18NTool.require_git_email_name(str(tmpdir))
assert 'please set name' in str(excinfo.value)
git.config('user.email', "*****@*****.**", **k)
git.config('user.name', "Your Name", **k)
assert i18n_tool.I18NTool.require_git_email_name(str(tmpdir))
def __init__(self):
self.__dict__ = self.__shared_state
self.content_dir = settings.WALIKI_DATA_DIR
os.chdir(self.content_dir)
if not os.path.isdir(os.path.join(self.content_dir, '.git')):
git.init()
git.config("user.email", settings.WALIKI_COMMITTER_EMAIL)
git.config("user.name", settings.WALIKI_COMMITTER_NAME)
self.git = git
def updateRepo(url, name, local_storage):
repo_path = os.path.join(local_storage, name)
if os.path.exists(repo_path) and os.path.exists(os.path.join(repo_path, '.git')):
git.reset(hard= True, _cwd= repo_path)
git.pull(_cwd= repo_path)
else:
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
git.clone(url, name, _cwd= local_storage)
git.config('core.fileMode', 'false', _cwd= repo_path)
git.config('core.sharedRepository', 'true', _cwd= repo_path)
def set_remote(self, chdir=False):
# init
remote = WALIKI_DATA_DIR + '_remote'
git.clone(WALIKI_DATA_DIR, remote)
os.chdir(remote)
git.config("user.email", '*****@*****.**')
git.config("user.name", 'I am me, yo')
os.chdir(WALIKI_DATA_DIR)
git.remote('add', 'origin', remote)
if chdir:
os.chdir(remote)
return remote
def set_remote(self, chdir=False):
# init
remote = WALIKI_DATA_DIR + '_remote'
git.clone(WALIKI_DATA_DIR, remote)
os.chdir(remote)
git.config("user.email", '*****@*****.**')
git.config("user.name", 'I am me, yo')
os.chdir(WALIKI_DATA_DIR)
git.remote('add', 'origin', remote)
if chdir:
os.chdir(remote)
return remote
def run_status(args, manifest, env, subparser):
if not manifest.layers:
return
layer_name_max_length = max(
map(lambda l: len(l.submodule), manifest.layers))
for layer in manifest.layers:
if layer.submodule == '.':
url = git.config('remote.origin.url')
else:
with pushd(layer.path):
url = git.config('remote.origin.url')
print("%%-%ds %%s" % (layer_name_max_length) %
(layer.submodule, url.strip()))
return
def run_status(args, manifest, env, subparser):
if not manifest.layers:
return
layer_name_max_length = max(map(lambda l: len(l.submodule),
manifest.layers))
for layer in manifest.layers:
if layer.submodule == '.':
url = git.config('remote.origin.url')
else:
with pushd(layer.path):
url = git.config('remote.origin.url')
print("%%-%ds %%s"%(layer_name_max_length)%(
layer.submodule, url.strip()))
return
def initializeDir(path, user, repo = None):
if not os.path.exists(path):
print('making directories')
os.makedirs(path)
os.chdir(path)
if os.path.exists('.git/'):
print('git respository already initialized')
return
print('initializing git repository')
git.init()
os.chmod(path + '/.git/', 0b111000000)
git.config('--local', 'user.name', '"' + user['name'] + '"')
git.config('--local', 'user.email', '"' + user['email'] + '"')
if repo:
git.remote.add('origin', repo)
def initializeDir(path, user, repo=None):
if not os.path.exists(path):
print('making directories')
os.makedirs(path)
os.chdir(path)
if os.path.exists('.git/'):
print('git respository already initialized')
return
print('initializing git repository')
git.init()
os.chmod(path + '/.git/', 0b111000000)
git.config('--local', 'user.name', '"' + user['name'] + '"')
git.config('--local', 'user.email', '"' + user['email'] + '"')
if repo:
git.remote.add('origin', repo)
def test_update_docs(self, tmpdir, caplog):
k = {"_cwd": str(tmpdir)}
git.init(**k)
git.config("user.email", "*****@*****.**", **k)
git.config("user.name", "Your Name", **k)
os.makedirs(join(str(tmpdir), "docs/includes"))
touch("docs/includes/l10n.txt", **k)
git.add("docs/includes/l10n.txt", **k)
git.commit("-m", "init", **k)
i18n_tool.I18NTool().main(
["--verbose", "update-docs", "--docs-repo-dir",
str(tmpdir)])
assert "l10n.txt updated" in caplog.text
caplog.clear()
i18n_tool.I18NTool().main(
["--verbose", "update-docs", "--docs-repo-dir",
str(tmpdir)])
assert "l10n.txt already up to date" in caplog.text
def test_update_docs(self, tmpdir, caplog):
k = {'_cwd': str(tmpdir)}
git.init(**k)
git.config('user.email', "*****@*****.**", **k)
git.config('user.name', "Your Name", **k)
os.mkdir(join(str(tmpdir), 'includes'))
touch('includes/l10n.txt', **k)
git.add('includes/l10n.txt', **k)
git.commit('-m', 'init', **k)
i18n_tool.I18NTool().main(
['--verbose', 'update-docs', '--documentation-dir',
str(tmpdir)])
assert 'l10n.txt updated' in caplog.text
caplog.clear()
i18n_tool.I18NTool().main(
['--verbose', 'update-docs', '--documentation-dir',
str(tmpdir)])
assert 'l10n.txt already up to date' in caplog.text
def test_clone_git_repo(self, mock_working_dir):
# Create a fake temp repo w/ commit
git_dir = tempfile.mkdtemp(prefix='git')
git.init(git_dir)
try:
f = open(os.path.join(git_dir, 'foo.txt'), 'w')
f.write('blah')
except IOError:
assert False
finally:
f.close()
cwd = os.getcwd()
os.chdir(git_dir)
git.add('foo.txt')
if os.environ.get('TRAVIS_CI') == 'true':
git.config('--global', 'user.email',
'"*****@*****.**"')
git.config('--global', 'user.name',
'"Testing on Travis CI"')
git.commit('-m "Added foo.txt"')
os.chdir(cwd)
# Fake Template Path
mock_working_dir.return_value = tempfile.gettempdir()
# Now attempt to clone but patch for Exception throw
self.config.template = 'git+' + git_dir
self.config._tpl = git_dir
t = Template(self.config)
t.copy_template()
# Clean up
rmtree(t.project_root)
self.assertFalse(os.path.isdir(t.config.template))
self.assertFalse(self.config.cli_opts.error.called)
def test_clone_git_repo(self, mock_working_dir):
# Create a fake temp repo w/ commit
git_dir = tempfile.mkdtemp(prefix='git')
git.init(git_dir)
try:
f = open(os.path.join(git_dir, 'foo.txt'), 'w')
f.write('blah')
except IOError:
assert False
finally:
f.close()
cwd = os.getcwd()
os.chdir(git_dir)
git.add('foo.txt')
if os.environ.get('TRAVIS_CI') == 'true':
git.config('--global', 'user.email',
'"*****@*****.**"')
git.config('--global', 'user.name', '"Testing on Travis CI"')
git.commit('-m "Added foo.txt"')
os.chdir(cwd)
# Fake Template Path
mock_working_dir.return_value = tempfile.gettempdir()
# Now attempt to clone but patch for Exception throw
self.config.template = 'git+' + git_dir
self.config._tpl = git_dir
t = Template(self.config)
t.copy_template()
# Clean up
rmtree(t.project_root)
self.assertFalse(os.path.isdir(t.config.template))
self.assertFalse(self.config.cli_opts.error.called)
def run(self):
for file_type in FILE_TYPES:
key = 'hooks.{}.{}'.format(self.name, file_type)
hooks = git.config(key).strip('\\n').split(',')
for hook in hooks:
command = Command.from_name(hook)()
sys.stdout.write(highlight('{}'.format(command.output)))
sys.stdout.flush()
try:
command.run(files=self.files)
except ErrorReturnCode, e:
print red(' ✗')
print ' ', '\n '.join([e.stderr, e.stdout])
self.result = 1
break
else:
print green(' ✔ ')
if self.result == 1:
break
def setUp(self):
super(TestEndToEnd, self).setUp('gl-e2e-test')
gl.init()
# Disable colored output so that we don't need to worry about ANSI escape
# codes
git.config('color.ui', False)
# Disable paging so that we don't have to use sh's _tty_out option, which is
# not available on pbs
if sys.platform != 'win32':
git.config('core.pager', 'cat')
else:
# On Windows, we need to call 'type' through cmd.exe (with 'cmd'). The /C
# is so that the command window gets closed after 'type' finishes
git.config('core.pager', 'cmd /C type')
utils.set_test_config()
def install_hooks(file_type):
"""
This is a fake install.
All it does is write the list of hooks that
need to be run to your git config file
"""
hooks = FILE_TYPE_HOOKS[file_type]
for hook in hooks.keys():
try:
base_key = 'hooks.{hook_type}'.format(hook_type=hook)
key = '{base_key}.{key_name}'.format(base_key=base_key, key_name='placeholder')
git.config('--get', key)
except ErrorReturnCode_1:
logging.info('This hook has not been installed')
continue
else:
key = '{base_key}.{key_name}'.format(base_key=base_key, key_name=file_type)
try:
git.config('--get', key)
except ErrorReturnCode_1:
git.config('--add', key, hooks[hook])
def setUp(self):
super(TestEndToEnd, self).setUp('gl-e2e-test')
gl.init()
git.config('color.ui', False)
utils.set_test_config()
def test_update_from_weblate(self, tmpdir, caplog):
d = str(tmpdir)
for repo in ('i18n', 'securedrop'):
os.mkdir(join(d, repo))
k = {'_cwd': join(d, repo)}
git.init(**k)
git.config('user.email', '*****@*****.**', **k)
git.config('user.name', 'Loïc Nordhøy', **k)
touch('README.md', **k)
git.add('README.md', **k)
git.commit('-m', 'README', 'README.md', **k)
for o in os.listdir(join(self.dir, 'i18n')):
f = join(self.dir, 'i18n', o)
if os.path.isfile(f):
shutil.copyfile(f, join(d, 'i18n', o))
else:
shutil.copytree(f, join(d, 'i18n', o))
k = {'_cwd': join(d, 'i18n')}
git.add('securedrop', 'install_files', **k)
git.commit('-m', 'init', '-a', **k)
git.checkout('-b', 'i18n', 'master', **k)
def r():
return "".join([str(l) for l in caplog.records])
#
# de_DE is not amount the supported languages, it is not taken
# into account despite the fact that it exists in weblate.
#
caplog.clear()
i18n_tool.I18NTool().main([
'--verbose',
'update-from-weblate',
'--root', join(str(tmpdir), 'securedrop'),
'--url', join(str(tmpdir), 'i18n'),
'--supported-languages', 'nl',
])
assert 'l10n: updated Dutch (nl)' in r()
assert 'l10n: updated German (de_DE)' not in r()
#
# de_DE is added but there is no change in the nl translation
# therefore nothing is done for nl
#
caplog.clear()
i18n_tool.I18NTool().main([
'--verbose',
'update-from-weblate',
'--root', join(str(tmpdir), 'securedrop'),
'--url', join(str(tmpdir), 'i18n'),
'--supported-languages', 'nl,de_DE',
])
assert 'l10n: updated Dutch (nl)' not in r()
assert 'l10n: updated German (de_DE)' in r()
#
# nothing new for nl or de_DE: nothing is done
#
caplog.clear()
i18n_tool.I18NTool().main([
'--verbose',
'update-from-weblate',
'--root', join(str(tmpdir), 'securedrop'),
'--url', join(str(tmpdir), 'i18n'),
'--supported-languages', 'nl,de_DE',
])
assert 'l10n: updated Dutch (nl)' not in r()
assert 'l10n: updated German (de_DE)' not in r()
message = str(git('--no-pager', '-C', 'securedrop', 'show',
_cwd=d, _encoding='utf-8'))
assert "Loïc" in message
#
# an update is done to nl in weblate
#
k = {'_cwd': join(d, 'i18n')}
f = 'securedrop/translations/nl/LC_MESSAGES/messages.po'
sed('-i', '-e', 's/inactiviteit/INACTIVITEIT/', f, **k)
git.add(f, **k)
git.config('user.email', '*****@*****.**', **k)
git.config('user.name', 'Someone Else', **k)
git.commit('-m', 'translation change', f, **k)
k = {'_cwd': join(d, 'securedrop')}
git.config('user.email', '*****@*****.**', **k)
git.config('user.name', 'Someone Else', **k)
#
# the nl translation update from weblate is copied
# over.
#
caplog.clear()
i18n_tool.I18NTool().main([
'--verbose',
'update-from-weblate',
'--root', join(str(tmpdir), 'securedrop'),
'--url', join(str(tmpdir), 'i18n'),
'--supported-languages', 'nl,de_DE',
])
assert 'l10n: updated Dutch (nl)' in r()
assert 'l10n: updated German (de_DE)' not in r()
message = str(git('--no-pager', '-C', 'securedrop', 'show',
_cwd=d))
assert "Someone Else" in message
assert "Loïc" not in message
def test_update_from_weblate(self, tmpdir, caplog):
d = str(tmpdir)
for repo in ("i18n", "securedrop"):
os.mkdir(join(d, repo))
k = {"_cwd": join(d, repo)}
git.init(**k)
git.config("user.email", "*****@*****.**", **k)
git.config("user.name", "Loïc Nordhøy", **k)
touch("README.md", **k)
git.add("README.md", **k)
git.commit("-m", "README", "README.md", **k)
for o in os.listdir(join(self.dir, "i18n")):
f = join(self.dir, "i18n", o)
if os.path.isfile(f):
shutil.copyfile(f, join(d, "i18n", o))
else:
shutil.copytree(f, join(d, "i18n", o))
k = {"_cwd": join(d, "i18n")}
git.add("securedrop", "install_files", **k)
git.commit("-m", "init", "-a", **k)
git.checkout("-b", "i18n", "master", **k)
def r():
return "".join([str(l) for l in caplog.records])
#
# de_DE is not amount the supported languages, it is not taken
# into account despite the fact that it exists in weblate.
#
caplog.clear()
i18n_tool.I18NTool().main([
"--verbose",
"update-from-weblate",
"--root",
join(str(tmpdir), "securedrop"),
"--url",
join(str(tmpdir), "i18n"),
"--supported-languages",
"nl",
])
assert "l10n: updated Dutch (nl)" in r()
assert "l10n: updated German (de_DE)" not in r()
#
# de_DE is added but there is no change in the nl translation
# therefore nothing is done for nl
#
caplog.clear()
i18n_tool.I18NTool().main([
"--verbose",
"update-from-weblate",
"--root",
join(str(tmpdir), "securedrop"),
"--url",
join(str(tmpdir), "i18n"),
"--supported-languages",
"nl,de_DE",
])
assert "l10n: updated Dutch (nl)" not in r()
assert "l10n: updated German (de_DE)" in r()
#
# nothing new for nl or de_DE: nothing is done
#
caplog.clear()
i18n_tool.I18NTool().main([
"--verbose",
"update-from-weblate",
"--root",
join(str(tmpdir), "securedrop"),
"--url",
join(str(tmpdir), "i18n"),
"--supported-languages",
"nl,de_DE",
])
assert "l10n: updated Dutch (nl)" not in r()
assert "l10n: updated German (de_DE)" not in r()
message = str(
git("--no-pager",
"-C",
"securedrop",
"show",
_cwd=d,
_encoding="utf-8"))
assert "Loïc" in message
#
# an update is done to nl in weblate
#
k = {"_cwd": join(d, "i18n")}
f = "securedrop/translations/nl/LC_MESSAGES/messages.po"
sed("-i", "-e", "s/inactiviteit/INACTIVITEIT/", f, **k)
git.add(f, **k)
git.config("user.email", "*****@*****.**", **k)
git.config("user.name", "Someone Else", **k)
git.commit("-m", "translation change", f, **k)
k = {"_cwd": join(d, "securedrop")}
git.config("user.email", "*****@*****.**", **k)
git.config("user.name", "Someone Else", **k)
#
# the nl translation update from weblate is copied
# over.
#
caplog.clear()
i18n_tool.I18NTool().main([
"--verbose",
"update-from-weblate",
"--root",
join(str(tmpdir), "securedrop"),
"--url",
join(str(tmpdir), "i18n"),
"--supported-languages",
"nl,de_DE",
])
assert "l10n: updated Dutch (nl)" in r()
assert "l10n: updated German (de_DE)" not in r()
message = str(git("--no-pager", "-C", "securedrop", "show", _cwd=d))
assert "Someone Else" in message
assert "Loïc" not in message
def add_to_blacklist(**kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--global", "--get", "user.name") == "":
return (False, "Tell someone to run `git config --global user.name \"SmokeDetector\"`")
if git.config("--global", "--get", "user.name") == "":
return (False, "Tell someone to run `git config --global user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!")
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character != "\n":
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push()
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist),
"body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n"
u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist,
item_to_blacklist, ms_search_option,
item_to_blacklist.replace(" ", "+"),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
print(response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(
response.json()["html_url"]))
except KeyError:
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
git.checkout("deploy") # Return to deploy to await CI.
return (True, "Blacklisted {0}".format(item_to_blacklist))
def setUp(self):
super(TestEndToEnd, self).setUp('gl-e2e-test')
gl.init()
git.config('color.ui', False)
utils.set_test_config()
def add_to_blacklist(**kwargs):
if 'windows' in str(platform.platform()).lower():
log('warning', "Git support not available in Windows.")
return (False, "Git support not available in Windows.")
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--global", "--get", "user.name", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config --global user.name \"SmokeDetector\"`")
if git.config("--global", "--get", "user.email", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config --global user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!")
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character != "\n":
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist),
"body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n"
u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist,
item_to_blacklist, ms_search_option,
item_to_blacklist.replace(" ", "+"),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(
response.json()["html_url"]))
except KeyError:
git.checkout("deploy") # Return to deploy
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
git.checkout("deploy") # Return to deploy to await CI.
return (True, "Blacklisted {0}".format(item_to_blacklist))
def set_git_author_info(name: str, email: str):
git.config("user.name", name)
git.config("user.email", email)
def set_ident(self, name, email):
git.config('user.email', email)
git.config('user.name', name)
def set_test_config():
git.config('user.name', 'test')
git.config('user.email', '*****@*****.**')
class WOVar():
"""Intialization of core variables"""
# WordOps version
wo_version = "3.10.1"
# WordOps packages versions
wo_wp_cli = "2.3.0"
wo_adminer = "4.7.3"
wo_phpmyadmin = "4.9.1"
wo_extplorer = "2.1.13"
wo_dashboard = "1.2"
# Get WPCLI path
wo_wpcli_path = '/usr/local/bin/wp'
# Current date and time of System
wo_date = datetime.now().strftime('%d%b%Y-%H-%M-%S')
# WordOps core variables
# linux distribution
wo_distro = linux_distribution(full_distribution_name=False)[0].lower()
wo_platform_version = linux_distribution(
full_distribution_name=False)[1].lower()
# distro codename (bionic, xenial, stretch ...)
wo_platform_codename = linux_distribution(
full_distribution_name=False)[2].lower()
# Get timezone of system
if os.path.isfile('/etc/timezone'):
with open("/etc/timezone", mode='r', encoding='utf-8') as tzfile:
wo_timezone = tzfile.read().replace('\n', '')
if wo_timezone == "Etc/UTC":
wo_timezone = "UTC"
else:
wo_timezone = "Europe/Amsterdam"
# Get FQDN of system
wo_fqdn = getfqdn()
# WordOps default webroot path
wo_webroot = '/var/www/'
# WordOps default renewal SSL certificates path
wo_ssl_archive = '/etc/letsencrypt/renewal'
# WordOps default live SSL certificates path
wo_ssl_live = '/etc/letsencrypt/live'
# PHP user
wo_php_user = '******'
# WordOps git configuration management
config = configparser.ConfigParser()
config.read(os.path.expanduser("~") + '/.gitconfig')
try:
wo_user = config['user']['name']
wo_email = config['user']['email']
except Exception:
print("WordOps (wo) require an username & and an email "
"address to configure Git (used to save server configurations)")
print("Your informations will ONLY be stored locally")
wo_user = input("Enter your name: ")
while wo_user == "":
print("Unfortunately, this can't be left blank")
wo_user = input("Enter your name: ")
wo_email = input("Enter your email: ")
while not match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$",
wo_email):
print("Whoops, seems like you made a typo - "
"the e-mailaddress is invalid...")
wo_email = input("Enter your email: ")
git.config("--global", "user.name", "{0}".format(wo_user))
git.config("--global", "user.email", "{0}".format(wo_email))
if not os.path.isfile('/root/.gitconfig'):
copy2(os.path.expanduser("~") + '/.gitconfig', '/root/.gitconfig')
# MySQL hostname
wo_mysql_host = ""
config = configparser.RawConfigParser()
if os.path.exists('/etc/mysql/conf.d/my.cnf'):
cnfpath = "/etc/mysql/conf.d/my.cnf"
else:
cnfpath = os.path.expanduser("~") + "/.my.cnf"
if [cnfpath] == config.read(cnfpath):
try:
wo_mysql_host = config.get('client', 'host')
except configparser.NoOptionError:
wo_mysql_host = "localhost"
else:
wo_mysql_host = "localhost"
# WordOps stack installation variables
# Nginx repo and packages
if wo_distro == 'ubuntu':
wo_nginx_repo = "ppa:wordops/nginx-wo"
else:
if wo_distro == 'debian':
if wo_platform_codename == 'jessie':
wo_deb_repo = "Debian_8.0"
elif wo_platform_codename == 'stretch':
wo_deb_repo = "Debian_9.0"
elif wo_platform_codename == 'buster':
wo_deb_repo = "Debian_10"
elif wo_distro == 'raspbian':
if wo_platform_codename == 'stretch':
wo_deb_repo = "Raspbian_9.0"
elif wo_platform_codename == 'buster':
wo_deb_repo = "Raspbian_10"
# debian/raspbian nginx repository
wo_nginx_repo = ("deb http://download.opensuse.org"
"/repositories/home:"
"/virtubox:/WordOps/{0}/ /".format(wo_deb_repo))
wo_nginx = ["nginx-custom", "nginx-wo"]
wo_nginx_key = '188C9FB063F0247A'
wo_php = [
"php7.2-fpm", "php7.2-curl", "php7.2-gd", "php7.2-imap",
"php7.2-readline", "php7.2-common", "php7.2-recode", "php7.2-cli",
"php7.2-mbstring", "php7.2-intl", "php7.2-bcmath", "php7.2-mysql",
"php7.2-opcache", "php7.2-zip", "php7.2-xml", "php7.2-soap"
]
wo_php73 = [
"php7.3-fpm", "php7.3-curl", "php7.3-gd", "php7.3-imap",
"php7.3-readline", "php7.3-common", "php7.3-recode", "php7.3-cli",
"php7.3-mbstring", "php7.3-intl", "php7.3-bcmath", "php7.3-mysql",
"php7.3-opcache", "php7.3-zip", "php7.3-xml", "php7.3-soap"
]
wo_php_extra = [
"php-memcached", "php-imagick", "graphviz", "php-xdebug",
"php-msgpack", "php-redis"
]
wo_mysql = ["mariadb-server", "percona-toolkit"]
if wo_distro == 'raspbian':
wo_mysql = wo_mysql + ["python3-mysqldb"]
else:
if wo_platform_codename == 'jessie':
wo_mysql = wo_mysql + ["python3-mysql.connector"]
else:
wo_mysql = wo_mysql + ["python3-mysqldb", "mariadb-backup"]
wo_mysql_client = ["mariadb-client"]
if wo_platform_codename == 'jessie':
wo_mysql_client = wo_mysql_client + ["python3-mysqldb"]
else:
wo_mysql_client = wo_mysql_client + ["python3-mysql.connector"]
wo_fail2ban = ["fail2ban"]
wo_clamav = ["clamav", "clamav-freshclam"]
# Redis repo details
if wo_distro == 'ubuntu':
wo_php_repo = "ppa:ondrej/php"
wo_redis_repo = ("ppa:chris-lea/redis-server")
wo_goaccess_repo = ("ppa:alex-p/goaccess")
wo_mysql_repo = ("deb [arch=amd64,ppc64el] "
"http://mariadb.mirrors.ovh.net/MariaDB/repo/"
"10.3/ubuntu {codename} main".format(
codename=wo_platform_codename))
else:
wo_php_repo = (
"deb https://packages.sury.org/php/ {codename} main".format(
codename=wo_platform_codename))
wo_php_key = 'AC0E47584A7A714D'
wo_redis_repo = (
"deb https://packages.sury.org/php/ {codename} all".format(
codename=wo_platform_codename))
wo_mysql_repo = ("deb [arch=amd64,ppc64el] "
"http://mariadb.mirrors.ovh.net/MariaDB/repo/"
"10.3/debian {codename} main".format(
codename=wo_platform_codename))
wo_redis = ['redis-server']
# Repo path
wo_repo_file = "wo-repo.list"
wo_repo_file_path = ("/etc/apt/sources.list.d/" + wo_repo_file)
# Application dabase file path
basedir = os.path.abspath(os.path.dirname('/var/lib/wo/'))
wo_db_uri = 'sqlite:///' + os.path.join(basedir, 'dbase.db')
def __init__(self):
pass
def set_test_config():
git.config('user.name', 'test')
git.config('user.email', '*****@*****.**')