def commit(self, objects, message):
# validate commit message
if not message or not isinstance(message, basestring):
raise ValueError(
"Commit message should not be empty or not string")
env = os.environ.copy()
env.update({
'GIT_WORK_TREE': self.repo,
'GIT_DIR': '%s/.git' % self.repo,
})
git.gc("--prune", _env=env)
git.checkout("HEAD", _env=env)
# pull and push from and to the remote
git.pull("origin", "master", _env=env)
for obj in objects:
git.add("-A", obj, _env=env)
try:
git.commit("-m", message, _env=env)
except Exception:
pass
git.push(_env=env)
def pull(self, remote, env={}, branch=None):
"""
Pull a branch from a given remote
Given a remote, env and branch, pull branch
from remote and add the environment variables
in the env dict to the environment of the
"git pull" command.
If no branch is given, the current branch
will be updated.
"""
os.chdir(self.repo)
if branch:
branch_to_pull = branch
else:
branch_to_pull = self.current_branch()
# if there is no PKEY, we don't need to override env
# We are explicit about what we are pushing, since the default behavior
# is different in different versions of Git and/or by configuration
if env["PKEY"]:
new_env = os.environ.copy()
new_env.update(env)
git.pull(remote, branch_to_pull, _env=new_env)
else:
git.pull(remote, branch_to_pull)
new_sha = git("rev-parse", "HEAD")
return new_sha.strip()
def commit(self, objects, message):
# validate commit message
if not message or not isinstance(message, basestring):
raise ValueError("Commit message should not be empty or not string")
env = os.environ.copy()
env.update({
'GIT_WORK_TREE': self.repo,
'GIT_DIR': '%s/.git' % self.repo,
})
git.gc("--prune", _env=env)
git.checkout("HEAD", _env=env)
# pull and push from and to the remote
git.pull("origin", "master", _env=env)
for obj in objects:
git.add("-A", obj, _env=env)
try:
git.commit("-m", message, _env=env)
except Exception:
pass
git.push(_env=env)
def pull(self, remote, env={}, branch=None):
"""
Pull a branch from a given remote
Given a remote, env and branch, pull branch
from remote and add the environment variables
in the env dict to the environment of the
"git pull" command.
If no branch is given, the current branch
will be updated.
"""
os.chdir(self.repo)
if branch:
branch_to_pull = branch
else:
branch_to_pull = self.current_branch()
# if there is no PKEY, we don't need to override env
# We are explicit about what we are pushing, since the default behavior
# is different in different versions of Git and/or by configuration
if env["PKEY"]:
new_env = os.environ.copy()
new_env.update(env)
git.pull(remote, branch_to_pull, _env=new_env)
else:
git.pull(remote, branch_to_pull)
new_sha = git("rev-parse","HEAD")
return new_sha.strip()
def _pull_repo(self, repo, key):
path = os.path.join(self._catalog_config.catalog_dir, repo['name'])
with utils.cd(path):
git.pull('--force',
'origin',
'{0}:{0}'.format(repo['branch']),
_env=self.__git_env(key))
def __init__(self, repo):
if path.exists(self.repo_path):
os.chdir(self.repo_path)
git.pull('-f', '-u', 'origin', 'master')
else:
git.clone(repo, self.REPO_PATH)
os.chdir(self.repo_path)
git.submodule('update', '--init', self.ROLES)
self._metadata = dict()
def _release(language, message, channel):
print message, "...",
if _is_dirty():
sys.exit("Repo must be in clean state before deploying. Please commit changes.")
_generate_yaml(language, channel)
if _is_dirty():
git.add('.travis.yml')
git.commit(m=message, allow_empty=True)
git.pull(rebase=True)
git.push()
print "done."
def updateRepo(url, name, local_storage):
repo_path = os.path.join(local_storage, name)
if os.path.exists(repo_path) and os.path.exists(os.path.join(repo_path, '.git')):
git.reset(hard= True, _cwd= repo_path)
git.pull(_cwd= repo_path)
else:
if os.path.exists(repo_path):
shutil.rmtree(repo_path)
git.clone(url, name, _cwd= local_storage)
git.config('core.fileMode', 'false', _cwd= repo_path)
git.config('core.sharedRepository', 'true', _cwd= repo_path)
def update_roles(self):
os.chdir('%s/%s' % (self.repo_path, self.ROLES))
git.pull('-f', '-u', 'origin', 'master')
os.chdir(self.repo_path)
message = '%s %s %s %s' % (self.UPDATE, self.ROLES, 'to',
self.get_submodule_hash(self.ROLES))
git.commit('-m', message, '.gitmodules', self.ROLES)
sys.stdout.write('Committed %s [%s]\n' % (self.ROLES, message))
try:
git.push('origin', 'master')
except sh.ErrorReturnCode_1:
pass
def migrate_for_path(packagepath, use_https):
_, name, _ = get_package_info(packagepath)
destination = os.path.join(destinationRoot, name)
if not os.path.exists(destination): # Only do conversion is path does not yet exist
print "Migrating {0} to {1} ...".format(packagepath, destination)
#import ipdb; ipdb.set_trace()
migrate_repo(packagepath, destination, use_https)
print "-" * 5, name, "done", "-"*5
else:
print "{0} already migrated at {1}. git svn rebase-ing instead to bring up to date".format(packagepath, destination)
git.pull("origin", "master")
git.svn.rebase()
git.push("origin", "master")
def _release(language, message, channel):
print message, "...",
if _is_dirty():
sys.exit(
"Repo must be in clean state before deploying. Please commit changes."
)
_generate_yaml(language, channel)
if _is_dirty():
git.add('.travis.yml')
git.commit(m=message, allow_empty=True)
git.pull(rebase=True)
git.push()
print "done."
def pull_repo():
"""
Pull the recent changes from origin master.
This function pulls all recent changes from the gitignore repository on
GitHub. It uses the module sh to invoke the git executable. If the git
executable cannot execute properly, an exception is thrown.
"""
try:
os.chdir(repo_dir)
git.pull()
except:
handle_exception()
return 0
def manage_checkout(self):
logger.info('manage checkout')
if not os.path.exists(self.checkouts_dir):
os.makedirs(self.checkouts_dir)
if not os.path.exists(self.checkout_dir):
logger.info('git clone %s %s' % (self.url, self.checkout_dir))
git.clone(self.url, self.checkout_dir)
else:
logger.info('git fetch -a')
git.fetch('-a', _cwd=self.checkout_dir)
logger.info('git pull --rebase')
git.pull('--rebase', _cwd=self.checkout_dir)
def migrate_repo(packagepath,
destination_path,
use_https,
authors="~/ros/fuerte/tue/authors.txt"):
"""#The process to migrate a project is:
# Create a new repository.
$ git svn clone <svn_url> <destination_path>
$ cd ~/ros/fuerte/tue/git/challenge_cleanup
$ git remote add origin <repo https url>
$ git pull origin master
$ git push origin master"""
authors = os.path.expanduser(authors)
svn_url = svnurl_for_path(packagepath)
language, name, description = get_package_info(packagepath)
try:
repo = create_repo(name, description, language=language)
except github.GithubException as e:
if e.status == 422:
cont = raw_input(
"The repo has an invalid field, it could already exist. Please verify and press 'c' to continue without first creating the repo: "
)
if 'c' in cont:
repo = tue.get_repo(name)
else:
sys.stderr.write("Could not migrate {0}".format(name))
return
print "git svn clone {0} {1} A={2}".format(svn_url, destination_path,
authors)
git.svn.clone(svn_url, destination_path, A=authors)
cd(destination_path)
if use_https:
git_url = repo.clone_url
else:
git_url = repo.ssh_url
print "git remote add origin {0}".format(git_url)
git.remote.add("origin", git_url)
print "git pull origin master"
git.pull("origin", "master")
print "git push origin master"
git.push("origin", "master")
def loop(self):
while True:
# time.sleep(20)
print "Pulling..."
git.pull()
latest_commit = self.__latest_commit()
if self.__current_commit == latest_commit:
# Sleep sometime, or next loop
continue
self.__current_commit = latest_commit
try:
make('test')
except sh.ErrorReturnCode:
print "Error"
continue
def loop(self):
while True:
# time.sleep(20)
print "Pulling..."
git.pull()
latest_commit = self.__latest_commit()
if self.__current_commit == latest_commit:
# Sleep sometime, or next loop
continue
self.__current_commit = latest_commit
try:
make('test')
except sh.ErrorReturnCode:
print "Error"
continue
def migrate_for_path(packagepath, use_https):
_, name, _ = get_package_info(packagepath)
destination = os.path.join(destinationRoot, name)
if not os.path.exists(
destination): # Only do conversion is path does not yet exist
print "Migrating {0} to {1} ...".format(packagepath, destination)
#import ipdb; ipdb.set_trace()
migrate_repo(packagepath, destination, use_https)
print "-" * 5, name, "done", "-" * 5
else:
print "{0} already migrated at {1}. git svn rebase-ing instead to bring up to date".format(
packagepath, destination)
git.pull("origin", "master")
git.svn.rebase()
git.push("origin", "master")
def _pull(self):
work_branch = self._repository_branch_name
ret = git.get_current_branch_name(self._local_repo_dir)
if (not ret.succeeded) or (not ret.output):
logger.error("Failed to get current branch name. Reason: '{}'.".format(ret.message))
logger.error("Failed to pull: '{}'.".format(self._repository_name))
return False
current_branch = ret.output
if work_branch != current_branch:
logger.info("Local repo branch: Exprected: '{}', Current: '{}'.".format(work_branch, current_branch))
ret = git.checkout_branch(self._local_repo_dir, work_branch)
if ret.succeeded:
logger.info("Switched branch: '{}'.".format(work_branch))
else:
logger.error("Failed to switch branch: '{}'. Reason: '{}'.".format(work_branch, ret.message))
logger.error("Failed to pull: '{}'.".format(self._repository_name))
return False
logger.info("Start pulling...")
ret = git.pull(self._local_repo_dir)
if ret.succeeded:
logger.info("Pulled: '{}' ('{}').".format(self._repository_name, work_branch))
return True
else:
logger.error("Failed to pull: '{}' ('{}'). Reason: '{}'.".format(self._repository_name, work_branch, ret.message))
return False
def pull(self, username="******", password="******"):
"""pull to remote."""
def line_proc(line, stdin):
if "Username for" in line:
stdin.put(username + "\n")
elif "Password for" in line:
stdin.put(password + "\n")
else:
print "git-push: ", line.strip()
rcv = self.char_to_line(line_proc)
try:
# This is a super hack. See http://amoffat.github.io/sh/tutorials/2-interacting_with_processes.html
# for some explaination.
p = git.pull(_out=rcv, _out_bufsize=0, _tty_in=True)
p.exit_code
except ErrorReturnCode_128:
raise Exception(
"""Push to repository repository failed. You will need to store or cache credentials.
You can do this by using ssh, .netrc, or a credential maanger.
See: https://www.kernel.org/pub/software/scm/git/docs/gitcredentials.html"""
)
return True
def pull(self, username="******", password="******"):
'''pull to remote'''
def line_proc(line,stdin):
if "Username for" in line:
stdin.put(username+ "\n")
elif "Password for" in line:
stdin.put(password+ "\n")
else:
print "git-push: ", line.strip()
rcv = self.char_to_line(line_proc)
try:
# This is a super hack. See http://amoffat.github.io/sh/tutorials/2-interacting_with_processes.html
# for some explaination.
p = git.pull( _out=rcv, _out_bufsize=0, _tty_in=True)
p.exit_code
except ErrorReturnCode_128:
raise Exception("""Push to repository repository failed. You will need to store or cache credentials.
You can do this by using ssh, .netrc, or a credential maanger.
See: https://www.kernel.org/pub/software/scm/git/docs/gitcredentials.html""")
return True
def create(version_number):
heading1("Creating new version based on Fedora " + version_number + "\n")
# Update git and create new version.
heading2("Updating master branch.")
print(git.checkout("master"))
print(git.pull()) # Bring branch current.
heading2("Creating new branch")
print(git.checkout("-b" + version_number)) # Create working branch.
# Get kickstart files.
heading2("Creating fedora-kickstarts directory\n")
mkdir("-p", (base_dir + "/fedora-kickstarts/"))
cd(base_dir + "/fedora-kickstarts/")
heading2("Downloading Fedora kickstart files.")
ks_base = "https://pagure.io/fedora-kickstarts/raw/f" \
+ version_number + "/f"
for file in ks_files:
file_path = ks_base + "/fedora-" + file
print ("Downloading " + file_path)
curl("-O", file_path)
def pull(url, path, remote=None):
here = str(pwd()).strip()
try:
cd(path)
except OSError:
print "path does not exist? {}".format(path)
cd(here)
return
try:
git.status()
except ErrorReturnCode_128:
print "{} is not a git repository!".format(path)
cd(here)
return
git.pull(remote or url, 'master')
git.checkout('-f')
cd(here)
def process_repo(dirname, repo):
"""Process one git repository"""
if not Config.verbose:
print("Process " + dirname)
if not os.path.isdir(dirname):
print("Clone " + repo["url"])
git.clone("--depth", "2", repo['url'])
os.chdir(dirname)
git.pull()
mds = glob.glob("./_posts/*/*.md")
Result.totalCount += len(mds)
prefix = "https://" + dirname + repo['prefix']
for path in mds:
process_article(path, prefix)
if not Config.verbose:
print()
os.chdir("..")
def migrate_repo(packagepath, destination_path, use_https, authors="~/ros/fuerte/tue/authors.txt"):
"""#The process to migrate a project is:
# Create a new repository.
$ git svn clone <svn_url> <destination_path>
$ cd ~/ros/fuerte/tue/git/challenge_cleanup
$ git remote add origin <repo https url>
$ git pull origin master
$ git push origin master"""
authors = os.path.expanduser(authors)
svn_url = svnurl_for_path(packagepath)
language, name, description = get_package_info(packagepath)
try:
repo = create_repo(name, description, language=language)
print "URLs: ".format(repo.clone_url, repo.ssh_url)
except github.GithubException as e:
if e.status == 422:
cont = raw_input(
"The repo has an invalid field, it could already exist. Please verify and press 'c' to continue without first creating the repo: ")
if 'c' in cont:
repo = tue.get_repo(name)
else:
sys.stderr.write("Could not migrate {0}".format(name))
return
print "git svn clone {0} {1} A={2}".format(svn_url, destination_path, authors)
git.svn.clone(svn_url, destination_path, A=authors)
cd(destination_path)
if use_https:
git_url = repo.clone_url
else:
git_url = repo.ssh_url
print "git remote add origin {0}".format(git_url)
git.remote.add("origin", git_url)
print "git pull origin master"
git.pull("origin", "master")
print "git push origin master"
git.push("origin", "master")
def deploy_code(repo):
logging.info("Deploying repo: %s, back2github: %s", repo["dir"], repo["sync"])
try:
logging.info("Change working directory to %s", repo["dir"])
sh.cd(repo["dir"])
logging.info("Update code, git pull origin")
git.pull("origin", _out=logging.info, _err=logging.error)
logging.info("done")
logging.info("Delete old branches, git fetch -p")
git.fetch("origin", "-p", _out=logging.info, _err=logging.error)
logging.info("done")
if repo["sync"]:
logging.info("Back2github, git push github")
git.push("github", _out=logging.info, _err=logging.error, _bg=True)
except Exception as e:
logging.info("Deploy error: %s", e)
def backup():
access_token = request.form["access_token"]
spreadsheet_id = request.form["spreadsheet_id"]
with cd("repo/"):
git.pull()
ods = export_as_ods(access_token, spreadsheet_id)
ods_path = write_bytes_to_file("clubs.ods", ods)
fods_path = convert_ods_to_fods(ods_path)
os.remove(ods_path)
# Only commit and push if any files have changed.
if git("ls-files", "-m"):
git.add(fods_path)
git.commit("-m", "Update spreadsheet.")
git.push()
return "Consider it done!"
def sync(path):
new_env = resetEnv()
logger.debug('Syncing {}'.format(path))
old_path = os.getcwd()
jobid = get_current_job().id
_open_console(jobid)
try:
os.chdir(path)
_log_console(jobid, 'Syncing project with Git.\n')
_l = lambda line: _log_console(jobid, str(line))
git.pull('--depth', 1, _out=_l, _err=_l, _env=new_env).wait()
except:
logger.error('Failed to sync project at {}'.format(path),
exc_info=True)
_close_console(jobid)
os.chdir(old_path)
def pull(self, branch, tag, sourceDir):
if not path.exists(sourceDir):
msgwarn("Source directory {} non-existing".format(sourceDir))
cp("-R", blenderSourceDir, sourceDir)
cd(sourceDir)
msgcmd("Update branch {}".format(branch))
#git.rebase("--abort")
git.clean("-fd", _out=debugSh)
git.checkout(".", _out=debugSh)
try:
git.checkout("-t", "origin/" + branch, _out=debugSh)
except:
pass
git.checkout(branch)
git.pull("origin", branch, "--rebase", _out=debugSh)
git.remote("prune", "origin")
git.pull("--rebase", _out=debugSh)
if tag is not "":
git.checkout(tag, _out=debugSh)
git.submodule.foreach("--recursive",
"git",
"checkout",
"master",
_out=debugSh)
git.submodule.foreach("git", "reset", "HEAD", "--hard", _out=debugSh)
git.submodule.update("--init", "--recursive", _out=debugSh)
git.submodule.foreach("--recursive",
"git",
"pull",
"--rebase",
"origin",
"master",
_out=debugSh)
def sync(path):
new_env = resetEnv()
logger.debug('Syncing {}'.format(path))
old_path = os.getcwd()
jobid = get_current_job().id
_open_console(jobid)
try:
os.chdir(path)
_log_console(jobid, 'Syncing project with Git.\n')
_l = lambda line: _log_console(jobid, str(line))
git.pull('--depth', 1, _out=_l, _err=_l, _env=new_env).wait()
except:
logger.error(
'Failed to sync project at {}'.format(path),
exc_info=True
)
_close_console(jobid)
os.chdir(old_path)
def clone_all_tue_packages(use_https):
cd(destinationRoot)
tue = g.get_organization("tue-robotics-lab")
tue_repos = tue.get_repos()
for repo in tue_repos:
if use_https:
git_url = repo.clone_url
else:
git_url = repo.ssh_url
print "{0}: ".format(repo.name),
try:
print "Cloning from {0}...".format(git_url),
git.clone(git_url, )
except sh.ErrorReturnCode_128 as e: # When the dir to clone already exists and is not empty, git pull this dir
print e
cd(repo.name)
print "Clone failed, pulling instead",
git.pull("origin", "master")
cd(destinationRoot)
print " Done"
def clone(repo):
global dest
dest = os.path.realpath(ARGS.outdir)
destrepo = os.path.join(dest, repo)
if not os.path.exists(destrepo):
os.chdir(dest)
# pp = git.clone("--progress", "[email protected]:%s/%s" %
# (ARGS.username, repo), _err=process_output,
# _out_bufsize=0)
git.clone("[email protected]:%s/%s" % (ARGS.username, repo),
_err=process_output,
_out_bufsize=0)
else:
# do git pull
os.chdir(destrepo)
pp = git.pull(_out=process_output, _out_bufsize=1)
pp.wait()
def update(self):
"""todo: Docstring for update
:return:
:rtype:
"""
logger.debug("")
rd = self.repo_dir
logger.debug("pkg path %s", rd)
if not rd:
print(
"unable to find pkg '%s'. %s" % (self.name, did_u_mean(self.name))
)
cwd = os.getcwd()
os.chdir(self.repo_dir)
logger.debug("cwd: %s, updating %s ", cwd, self.repo_dir)
try:
p = git.pull('--rebase', '--progress',
_out=self._sh_stdout('blue'),
_err=self._sh_stderr('red'))
p.wait()
except Exception as e:
pass
# logger.warn(e)
os.chdir(cwd)
# Update or install any dependancies before running the
# update script.
self.install_update_deps()
up = os.path.join(self.repo_dir, '_upkg', 'update')
if os.path.exists(up):
# We use subprocess instead of the sh module due to problems with
# runing shell scripts with sh
cwd = os.getcwd()
os.chdir(os.path.join(self.repo_dir, '_upkg'))
self.pr_info("Running update script for {} @ {}", self.name, up)
subprocess.check_call(up, shell=True)
os.chdir(cwd)
def update(self):
"""todo: Docstring for update
:return:
:rtype:
"""
logger.debug("")
rd = self.repo_dir
logger.debug("pkg path %s", rd)
if not rd:
print("unable to find pkg '%s'. %s" %
(self.name, did_u_mean(self.name)))
cwd = os.getcwd()
os.chdir(self.repo_dir)
logger.debug("cwd: %s, updating %s ", cwd, self.repo_dir)
try:
p = git.pull('--rebase',
'--progress',
_out=self._sh_stdout('blue'),
_err=self._sh_stderr('red'))
p.wait()
except Exception as e:
pass
# logger.warn(e)
os.chdir(cwd)
# Update or install any dependancies before running the
# update script.
self.install_update_deps()
up = os.path.join(self.repo_dir, '_upkg', 'update')
if os.path.exists(up):
# We use subprocess instead of the sh module due to problems with
# runing shell scripts with sh
cwd = os.getcwd()
os.chdir(os.path.join(self.repo_dir, '_upkg'))
self.pr_info("Running update script for {} @ {}", self.name, up)
subprocess.check_call(up, shell=True)
os.chdir(cwd)
def git_pull(remote, branch, file_path):
'''Pull remote changes'''
p = git.pull(remote, branch, _cwd=dir_path(file_path), _tty_out=False)
p.wait()
show_msg('Pull Done', 'Info')
def update_submodule(self, submodule_path):
os.chdir(submodule_path)
git.pull('-f', '-u', 'origin', 'master')
os.chdir(self.repo_path)
def _pull_repo(self, repo, key):
path = os.path.join(self._catalog_config.catalog_dir, repo['name'])
with utils.cd(path):
git.pull('--force', 'origin', '{0}:{0}'.format(repo['branch']),
_env=self.__git_env(key))
def add_to_blacklist(**kwargs):
if 'windows' in str(platform.platform()).lower():
log('warning', "Git support not available in Windows.")
return (False, "Git support not available in Windows.")
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--global", "--get", "user.name", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config --global user.name \"SmokeDetector\"`")
if git.config("--global", "--get", "user.email", _ok_code=[0, 1]) == "":
return (False, "Tell someone to run `git config --global user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!")
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character != "\n":
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
git.commit("--author='SmokeDetector <*****@*****.**>'",
"-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push("origin", "master")
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist),
"body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n"
u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist,
item_to_blacklist, ms_search_option,
item_to_blacklist.replace(" ", "+"),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
log('debug', response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
git.branch('-D', branch) # Delete the branch in the local git tree since we're done with it.
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(
response.json()["html_url"]))
except KeyError:
git.checkout("deploy") # Return to deploy
# Delete the branch in the local git tree, we'll create it again if the
# command is run again. This way, we keep things a little more clean in
# the local git tree
git.branch('-D', branch)
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
git.checkout("deploy") # Return to deploy to await CI.
return (True, "Blacklisted {0}".format(item_to_blacklist))
def pull(self, remote):
log = git.pull('-s', 'recursive', '-X', 'ours', remote,
'HEAD').stdout.decode('utf8')
return log
import base64
import os
from datetime import datetime
from sh import git
ips = []
__folder__ = os.path.split(__file__)[0]
if __folder__:
os.chdir(__folder__)
regex = re.compile(r"\/(.+)\/")
l = requests.get(
"https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf"
).text
print("load changes")
git.pull("--all")
git.reset("origin/master", "--hard")
git.pull()
print("Finish download")
for i in l.split("\n")[1:]:
if i and regex.search(i):
ips.append(regex.search(i).groups()[0])
with open("rawlist.txt", "w") as fp:
for i in ips:
fp.write(i + "\n")
with open("list.txt", 'w') as fp:
# s = "[AutoProxy 0.2.9]\n"
for i in ips:
def add_to_blacklist(**kwargs):
blacklist = kwargs.get("blacklist", "")
item_to_blacklist = kwargs.get("item_to_blacklist", "")
username = kwargs.get("username", "")
chat_profile_link = kwargs.get("chat_profile_link", "http://chat.stackexchange.com/users")
code_permissions = kwargs.get("code_permissions", False)
# Make sure git credentials are set up
if git.config("--global", "--get", "user.name") == "":
return (False, "Tell someone to run `git config --global user.name \"SmokeDetector\"`")
if git.config("--global", "--get", "user.name") == "":
return (False, "Tell someone to run `git config --global user.email \"[email protected]\"`")
if blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: blacklist")
if item_to_blacklist == "":
# If we broke the code, and this isn't assigned, error out before doing anything, but do
# so gracefully with a nice error message.
return (False, "Programming Error - Critical information missing for GitManager: item_to_blacklist")
item_to_blacklist = item_to_blacklist.replace("\s", " ")
if blacklist == "website":
blacklist_file_name = "blacklisted_websites.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "keyword":
blacklist_file_name = "bad_keywords.txt"
ms_search_option = "&body_is_regex=1&body="
elif blacklist == "username":
blacklist_file_name = "blacklisted_usernames.txt"
ms_search_option = "&username_is_regex=1&username="******"Invalid blacklist type specified, something has broken badly!")
git.checkout("master")
try:
git.pull()
except:
pass
# Check that we're up-to-date with origin (GitHub)
git.remote.update()
if git("rev-parse", "refs/remotes/origin/master").strip() != git("rev-parse", "master").strip():
return (False, "HEAD isn't at tip of origin's master branch")
# Check that blacklisted_websites.txt isn't modified locally. That could get ugly fast
if blacklist_file_name in git.status(): # Also ugly
return (False, "{0} is modified locally. This is probably bad.".format(blacklist_file_name))
# Add item to file
with open(blacklist_file_name, "a+") as blacklist_file:
last_character = blacklist_file.read()[-1:]
if last_character != "\n":
blacklist_file.write("\n")
blacklist_file.write(item_to_blacklist + "\n")
# Checkout a new branch (for PRs for non-code-privileged people)
branch = "auto-blacklist-{0}".format(str(time.time()))
git.checkout("-b", branch)
# Clear HEAD just in case
git.reset("HEAD")
git.add(blacklist_file_name)
git.commit("-m", u"Auto blacklist of {0} by {1} --autopull".format(item_to_blacklist, username))
if code_permissions:
git.checkout("master")
git.merge(branch)
git.push()
else:
git.push("origin", branch)
git.checkout("master")
if GlobalVars.github_username is None or GlobalVars.github_password is None:
return (False, "Tell someone to set a GH password")
payload = {"title": u"{0}: Blacklist {1}".format(username, item_to_blacklist),
"body": u"[{0}]({1}) requests the blacklist of the {2} {3}. See the Metasmoke search [here]"
"(https://metasmoke.erwaysoftware.com/search?utf8=%E2%9C%93{4}{5})\n"
u"<!-- METASMOKE-BLACKLIST-{6} {3} -->".format(username, chat_profile_link, blacklist,
item_to_blacklist, ms_search_option,
item_to_blacklist.replace(" ", "+"),
blacklist.upper()),
"head": branch,
"base": "master"}
response = requests.post("https://api.github.com/repos/Charcoal-SE/SmokeDetector/pulls",
auth=HTTPBasicAuth(GlobalVars.github_username, GlobalVars.github_password),
data=json.dumps(payload))
print(response.json())
try:
git.checkout("deploy") # Return to deploy, pending the accept of the PR in Master.
return (True, "You don't have code privileges, but I've [created a pull request for you]({0}).".format(
response.json()["html_url"]))
except KeyError:
# Error capture/checking for any "invalid" GH reply without an 'html_url' item,
# which will throw a KeyError.
if "bad credentials" in str(response.json()['message']).lower():
# Capture the case when GH credentials are bad or invalid
return (False, "Something is wrong with the GH credentials, tell someone to check them.")
else:
# Capture any other invalid response cases.
return (False, "A bad or invalid reply was received from GH, the message was: %s" %
response.json()['message'])
git.checkout("deploy") # Return to deploy to await CI.
return (True, "Blacklisted {0}".format(item_to_blacklist))
try:
persistent_arguments.remove('standby')
except:
pass # We're OK if the argument isn't in the list.
try:
ecode = sp.call(command + persistent_arguments, env=environ)
except KeyboardInterrupt:
# print "[NoCrash] KeyBoard Interrupt received.."
ecode = 6
if ecode == 3:
# print "[NoCrash] Pull in new updates."
if 'windows' not in str(platform.platform()).lower():
git.checkout('deploy')
git.pull()
git.submodule('update')
count = 0
crashcount = 0
elif ecode == 4:
# print "[NoCrash] Crashed."
count += 1
sleep(5)
if crashcount == 2:
# print "[NoCrash] Going to reverted state."
if 'windows' not in str(platform.platform()).lower():
git.checkout('HEAD~1')
def update_addon(self, addon):
addonPath = os.path.join(self.get_addon_path(), addon.name)
os.chdir(addonPath)
git.checkout('master')
git.pull()
def pull(self, remote):
log = git.pull('-s', 'recursive', '-X', 'ours', remote, 'HEAD').stdout.decode('utf8')
return log
def get_current_commit(repo_url):
source = source_path(repo_url)
os.chdir('{}'.format(source))
git.pull('origin', 'master')
commit_sha = git.log("-n1", "--pretty='%H'")
return get_raw_sha(commit_sha)
def manage(conf, args):
'''
Move a file to the base directory and leave a link pointing to its new
location in its place.
'''
# bail if the file is already a link
if os.path.islink(args.path):
raise ValueError('Unable to manage ' + color.cyan(args.path) +
" since it's already a link!")
# make sure the path is a descendant of the destination directory
if not util.is_descendant(args.path, conf['destination']):
raise ValueError("Unable to manage files that aren't descendants of " +
'the destination directory (' + color.cyan(conf['destination']) + ')')
# mark files that aren't direct descendants of the root as such
unrooted = os.path.dirname(args.path) != conf['destination']
# get the path of the file if it will be copied into the repo directory
dest_path = os.path.join(constants.REPO_DIR, os.path.basename(args.path))
# rename the file as appropriate to to its original name
dest_path, config_file_path = config.configify_file_name(dest_path)
# give unrooted files a config file path so they'll go to the correct place
if unrooted and config_file_path is None:
config_file_path = util.toggle_hidden(dest_path, True)
# bail if the file is already managed and we're not overwriting
dest_exists = os.path.exists(dest_path)
config_exists = (config_file_path is not None and
os.path.exists(config_file_path))
if (dest_exists or config_exists) and not args.force:
raise ValueError("Can't manage " + color.cyan(args.path) +
" since it already appears to be managed (use --force to override)")
# create a file config if necessary
# replace any existing dest file with a copy of the new one
util.rm(dest_path, force=True)
util.cp(args.path, dest_path, recursive=True)
# replace any existing config file with our new one
if config_file_path is not None:
util.rm(config_file_path, force=True)
# build a config for this file
file_config = config.normalize_file_config({
'paths': [args.path],
}, conf['destination'])
# create a config file from our config dict
with open(config_file_path, 'w') as f:
json.dump(file_config, f, indent=2)
# create a link to the new location, overwriting the old file
util.symlink(args.path, dest_path, overwrite=True)
print(color.cyan(args.path), 'copied and linked')
# add and commit the file to the repo if --save is specified
if args.save:
files = [color.cyan(os.path.basename(dest_path))]
if config_file_path:
files.append(color.cyan(os.path.basename(config_file_path)))
files = files.join(' and ')
print('Adding', files, 'to the repository...')
# move us to the current repo directory so all git commands start there
os.chdir(constants.REPO_DIR)
# alert the user if we have uncommitted changes (git exits non-0 in this case)
if git.diff(exit_code=True, quiet=True, _ok_code=(0, 1)).exit_code != 0:
raise ValueError('The repository has uncommitted changes - the '
'newly-managed file will have to be added to the repo manually.')
# add the new files to the staging area
git.add(dest_path)
if config_file_path is not None:
git.add(config_file_path)
print('Successfully added', files, 'to the repository')
print('Committing changes...')
# commit the file to the repository
commit_message = 'Manage %s' % os.path.basename(args.path)
git.commit(m=commit_message, quiet=True)
print('Commit successful!')
print('Pushing committed changes...')
# pull any changes down from upstream, then push our new addition
git.pull(rebase=True, quiet=True)
git.push(quiet=True)
print('Push successful!')
def tag(obj, tag_name, remote, yes):
current_branch = get_current_branch()
remote_url = git.remote("get-url", remote).strip()
gh, repo = obj
tag = split_version(tag_name)
tag_name = format_version(tag)
major, minor, fix = tag
with Spinner(f"Checking for milestone for tag {tag_name}"):
tag_milestone = None
for ms in repo.get_milestones(state="all"):
if ms.title == tag_name:
tag_milestone = ms
break
assert tag_milestone is not None, "Did not find milestone for tag"
release_branch_name = f"release/v{major}.{minor:>02}.X"
with Spinner("Refreshing branches"):
git.fetch(all=True, prune=True)
if fix == 0:
# new minor release
with Spinner(f"Checking out and updating {default_branch_name}"):
git.checkout(default_branch_name)
git.pull()
assert not check_branch_exists(
release_branch_name
), "For new minor: release branch CANNOT exist yet"
with Spinner(f"Creating {release_branch_name}"):
git.checkout("-b", release_branch_name)
else:
assert check_branch_exists(
release_branch_name), "For new fix: release brunch MUST exist"
with Spinner(f"Checking out {release_branch_name}"):
git.checkout(release_branch_name)
# we are not on release branch
version_file = Path("version_number")
assert version_file.exists(), "Version number file not found"
current_version_string = version_file.read_text()
print(f"Current version: [bold]{current_version_string}[/bold]")
if fix == 0:
assert current_version_string == "9.9.9", "Unexpected current version string found"
else:
assert current_version_string != f"{major}.{minor}.{fix-1}", "Unexpected current version string found"
version_string = f"{major}.{minor}.{fix}"
with Spinner(
f"Bumping version number in '{version_file}' to '{version_string}'"
):
with version_file.open("w") as fh:
fh.write(version_string)
with Spinner("Comitting"):
git.add(version_file)
git.commit(m=f"Bump version number to {version_string}")
with Spinner(f"Creating tag {tag_name}"):
git.tag(tag_name)
print(
f"I will now: push tag [bold green]{tag_name}[/bold green] and branch [bold green]{release_branch_name}[/bold green] to [bold]{remote_url}[/bold]"
)
if not confirm("Continue?", yes=yes):
raise SystemExit("Aborting")
with Spinner(f"Pushing branch {release_branch_name}"):
git.push("-u", remote, release_branch_name)
with Spinner(f"Pushing tag {tag_name}"):
git.push(remote, tag_name)
def pull(self):
'''pull to remote'''
git.pull(self.remote_url)
return True
def new_comment(author, email, site, url, article, message,
subscribe):
try:
logger.info("new comment received: author %s url %s subscribe %s"
% (author, url, subscribe))
if site and site[:4] != "http":
site = "http://" + site
# Create comment
now = datetime.now()
comment_list = (
'author: %s' % author,
'email: %s' % email,
'site: %s' % site,
'date: %s' % now.strftime("%Y-%m-%d %H:%M:%S"),
'url: %s' % url,
'article: %s' % article,
'',
'%s' % message,
''
)
comment = '\n'.join(comment_list)
logger.debug(comment)
# Git
branch_name = now.strftime("%Y%m%d%H%M%S%f")
if pecosys.get_config("git", "disabled"):
logger.debug("GIT usage disabled (debug mode)")
else:
git.checkout('master')
if pecosys.get_config("git", "remote"):
git.pull()
try:
git.checkout(branch_name)
except:
git.branch(branch_name)
git.checkout(branch_name)
comment_filename = 'comment-%s.md' % now.strftime("%Y%m%d-%H%M%S")
comment_pathname = '%s/%s/%s' % (pecosys.get_config('git', 'comment_path'),
now.year, comment_filename)
with open(comment_pathname, 'wb') as f:
f.write(comment.encode('UTF-8'))
git.add(comment_pathname)
git.commit('-m', 'new comment')
git.checkout('master')
# Render email body template
email_body = get_template('new_comment').render(url=url, comment=comment)
# Send email
mail(pecosys.get_config('post', 'from_email'),
pecosys.get_config('post', 'to_email'),
'[' + branch_name + '-' + article + ']', email_body)
# Reader subscribes to further comments
if subscribe and email:
subscribe_reader(email, article, url)
logger.debug("new comment processed ")
except:
logger.exception("new_comment failure")
from tester import *
from tracer import *
from xvfb import *
from sh import git, cd
import sys
display = Display()
cd(blenderSourceDir)
git.remote("prune", "origin")
git.pull("--rebase")
if "--branch" in sys.argv:
branch = sys.argv[sys.argv.index("--branch") + 1]
else:
branch = "master"
args = []
script = ""
if "basic" in sys.argv:
script = "BasicMain"
elif "error" in sys.argv:
script = "PythonErrorMain"
args = [pythonTestLogDir]
scriptFile = pythonMainScript + script + ".py"
if "valgrind" in sys.argv:
tester = Valgrind(branch, scriptFile, args)
elif "tracer" in sys.argv:
tester = BranchTracer(branch, "0000")
def git_pull(remote, branch, file_path):
'''Pull remote changes'''
p = git.pull(remote, branch, _cwd=dir_path(file_path), _tty_out=False)
p.wait()
show_msg('Pull Done', 'Info')
import requests
import re
import base64
import os
from datetime import datetime
from sh import git
ips = []
__folder__ = os.path.split(__file__)[0]
if __folder__:
os.chdir(__folder__)
regex = re.compile(r"\/(.+)\/")
l = requests.get("https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/accelerated-domains.china.conf").text
print("load changes")
git.pull("--all")
git.reset("origin/master", "--hard")
git.pull()
print("Finish download")
for i in l.split("\n")[1:]:
if i and regex.search(i):
ips.append(regex.search(i).groups()[0])
with open("rawlist.txt", "w") as fp:
for i in ips:
fp.write(i+"\n")
with open("list.txt", 'w') as fp:
# s = "[AutoProxy 0.2.9]\n"
def pull(self, remote):
log = git.pull("-s", "recursive", "-X", "ours", remote, "HEAD").stdout.decode("utf8")
return log
def patch(version, dry_run, gitlab):
project = gitlab.projects.get("acts/acts-core")
version = split_version(version)
milestone = find_milestone(version,
project.milestones.list(state="active"))
assert (milestone is not None
), f"Didn't find milestone for {version}. Is it closed already?"
branches = get_branches()
release_branch = "release/v{:d}.{:>02d}.X".format(*version)
version_file = Path() / "version_number"
tag_name = format_version(version)
if release_branch not in branches:
print("Release branch", release_branch, "does not exist. I'm bailing")
print("Will make new patch version tag %s from milestone %s on branch %s" %
(format_version(version), milestone.title, release_branch))
if click.confirm("Do you want to run local preparation?"):
with Spinner(
text=f"Checkout and update release branch {release_branch}"):
if not dry_run:
git.checkout(release_branch)
assert current_branch() == release_branch
git.pull()
with Spinner(text=f"Bumping version to {format_version(version)}"):
if not dry_run:
assert current_branch() == release_branch
with version_file.open("w") as fh:
fh.write(".".join(map(str, version)))
with Spinner(
text=
f"Committing bumped version on release branch {release_branch}"
):
if not dry_run:
git.add(str(version_file))
git.commit(message="Bump version to %s" %
".".join(map(str, version)))
with Spinner(
text=f"Creating local tag {tag_name} on {release_branch}"):
if not dry_run:
git.tag(tag_name)
print(f"You might want to run 'git push REMOTE {tag_name}'")
if click.confirm(f"Do you want me to try to push {release_branch}?"):
with Spinner(text=f"Pushing {release_branch}"):
if not dry_run:
git.push()
if click.confirm(f"Do you want me to close %{milestone.title}?"):
with Spinner(text=f"Closing milestone %{milestone.title}"):
if not dry_run:
milestone.state_event = "close"
milestone.save()