def prepare_repo(repos_dir, repo_name, repo_url):
# Confirm that a git repo exists at the given local directory. If it does
# not exist, then attempt to clone it. If it already exists, then
# fetch new branches and prune old branches.
log.debug("Preparing repo " + repo_name)
repo_path = os.path.join(repos_dir, repo_name)
dot_git_path = os.path.join(repo_path, '.git')
clone_the_repo = False
if not os.path.isdir(repo_path):
log.warning(f"Directory {repo_path} not found")
clone_the_repo = True
elif not os.path.isdir(dot_git_path):
# If we find a directory in our mounted volume that is not a git
# repo, then something is seriously wrong. Delete the directory and
# start again with a clean repo.
log.warning(f"Deleting invalid git repo found at {repo_path}")
shutil.rmtree(repo_path)
clone_the_repo = True
else:
log.debug(f"Directory {repo_path} is apparently a git repo")
if clone_the_repo:
log.info(f"Cloning {repo_name} ...")
os.chdir(repos_dir)
git.clone(repo_url)
else:
# The git repo exists, so bring it up to date
log.info(f"Fetching changes to {repo_name}")
os.chdir(repo_path)
git.fetch('--all')
return repo_path
def run(self, sorted_repos, versions_file):
self.log("`git checkout` repositories...")
versions = None
with open(versions_file, 'r') as f:
versions = json.load(f)
cd(self._basedir)
for repo in sorted_repos:
if repo not in versions:
self.log("skipping {0}, no version specified.".format(repo))
continue
where = versions[repo] # where to checkout
self.log("Checkout {0} -> {1}".format(repo, where))
with push_pop(repo):
git.fetch()
git.checkout("--quiet", where)
# just in case that we didn't just cloned but updated:
git.reset("--hard", where)
self.log("done checking out repos.")
def lookup_branch(self, branch_name):
if not stdout(git('ls-remote', '--heads', self.name, branch_name)):
return None
# The branch exists in the remote
git.fetch(self.git_remote.name, branch_name)
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name, pygit2.GIT_BRANCH_REMOTE)
return RemoteBranch(git_branch, self.gl_repo)
def lookup_branch(self, branch_name):
if not stdout(git('ls-remote', '--heads', self.name, branch_name)):
return None
# The branch exists in the remote
git.fetch(self.git_remote.name, branch_name)
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name, pygit2.GIT_BRANCH_REMOTE)
return RemoteBranch(git_branch, self.gl_repo)
def ensure_i18n_remote(self, args: argparse.Namespace) -> None:
"""
Make sure we have a git remote for the i18n repo.
"""
k = {"_cwd": args.root}
if b"i18n" not in git.remote(**k).stdout:
git.remote.add("i18n", args.url, **k)
git.fetch("i18n", **k)
def update(conf, args):
'''Apply updates from the upstream repository.'''
print('Checking for updates...')
# fetch changes from the canonical repo
git.fetch(constants.GIT_REMOTE, no_tags=True, quiet=True)
# get a list of the commit messages for the incoming changes
updates = git('--no-pager', 'log', '..FETCH_HEAD', oneline=True)
updates = [tuple(m.split(None, 1)) for m in updates.splitlines()]
# print out a list of the incoming updates
if len(updates) > 0:
print('Available updates:')
max_updates = 10
for commit, msg in updates[:max_updates]:
print(color.yellow('*'), msg)
# print a special message if too many updates are available
if len(updates) > max_updates:
print('...and', color.green(len(updates) - max_updates), 'more!')
print('Run `git log ..FETCH_HEAD` to see the full list')
# bail if we have uncommitted changes (git exits non-0 in this case)
if git.diff(exit_code=True, quiet=True, _ok_code=(0, 1)).exit_code != 0:
raise ValueError('The repository has uncommitted changes. Handle them, '
'then try updating again.')
print('Applying the update...')
# stash _all_ changes to the repo
git.stash(include_untracked=True, all=True, quiet=True)
# squash all the fetched commits together and merge them into master
git.merge('@{u}', squash=True, quiet=True)
# add a nice update commit that includes the latest upstream commit hash
commit_message = 'Update dotparty to %s' % updates[0][0]
git.commit(m=commit_message, quiet=True)
# TODO: if squash merge failed, roll back to the pre-update state and
# complain with instructions for the user to do their own update.
# un-stash all our old changes
git.stash('pop', quiet=True)
# push our changes back up to the remote
git.push(quiet=True)
print('Update successful!')
else:
print('Already up-to-date!')
def lookup_tag(self, tag_name):
tag_info = stdout(git('ls-remote', '--tags', self.name, tag_name))
if not tag_info:
return None
# The tag exists in the remote
git.fetch(self.git_remote.name, tag_name)
regex = re.compile(r'(.*)\trefs/tags/.*')
commit_id = regex.match(tag_info).group(1)
commit = self.gl_repo.git_repo.get(commit_id).peel(pygit2.GIT_OBJ_COMMIT)
return RemoteTag(self.git_remote.name, tag_name, commit)
def lookup_tag(self, tag_name):
tag_info = stdout(git('ls-remote', '--tags', self.name, tag_name))
if not tag_info:
return None
# The tag exists in the remote
git.fetch(self.git_remote.name, tag_name)
regex = re.compile(r'(.*)\trefs/tags/.*')
commit_id = regex.match(tag_info).group(1)
commit = self.gl_repo.git_repo.get(commit_id).peel(pygit2.GIT_OBJ_COMMIT)
return RemoteTag(self.git_remote.name, tag_name, commit)
def main():
options = json.load(sys.stdin)
github_remote_url = f"{get_github_url()}/{options['target_remote']}.git"
if len(list(pathlib.Path('.').iterdir())) > 0:
print("Found existing files in work directory", file=sys.stderr)
assert pathlib.Path('.git').exists(
), "if files are present in the work dir, it must be a git work tree"
remote = "origin"
if options["source_remote_name"] == remote:
remote = remote + "2"
add_or_set_git_remote(remote, github_remote_url)
print(f"Fetching from {github_remote_url}", file=sys.stderr)
git.fetch(remote)
print(
f"Checking out {options['target_branch']} from {remote}/{options['target_branch']}",
file=sys.stderr)
git.checkout("-B", options['target_branch'],
f"{remote}/{options['target_branch']}")
print(f"Cleaning work tree", file=sys.stderr)
git.reset("--hard", "HEAD")
git.clean("-fdx")
else:
print(f"Cloning {options['target_branch']} from {github_remote_url}",
file=sys.stderr)
git.clone("--branch", options['target_branch'], github_remote_url, ".")
if options['target_remote'] != options['source_remote']:
source_remote_name = options['source_remote_name']
add_or_set_git_remote(
source_remote_name,
f"{get_github_url()}/{options['source_remote']}.git")
print(
f"Fetching from {get_github_url()}/{options['source_remote']}.git",
file=sys.stderr)
git.fetch(source_remote_name)
set_git_author_info(f"GitHub Action {os.environ['GITHUB_ACTION']}",
"action@localhost")
try:
git("cherry-pick", options['source_commits'])
print(
f"Source commits ({options['source_commits']}) were successfully cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
except sh.ErrorReturnCode:
print(
f"Source commits ({options['source_commits']}) could not be cherry-picked "
f"onto {options['target_remote']}:{options['target_branch']}",
file=sys.stderr)
raise
def lookup_branch(self, branch_name):
if not stdout(git('ls-remote', '--heads', self.name, branch_name)):
return None
# The branch exists in the remote
git.fetch(self.git_remote.name, branch_name)
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name, pygit2.GIT_BRANCH_REMOTE)
# Make another check for the branch being None
# As observed in issue : https://github.com/sdg-mit/gitless/issues/211
if git_branch is None:
git.fetch(self.git_remote.name)
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name, pygit2.GIT_BRANCH_REMOTE)
return RemoteBranch(git_branch, self.gl_repo)
def manage_checkout(self):
logger.info('manage checkout')
if not os.path.exists(self.checkouts_dir):
os.makedirs(self.checkouts_dir)
if not os.path.exists(self.checkout_dir):
logger.info('git clone %s %s' % (self.url, self.checkout_dir))
git.clone(self.url, self.checkout_dir)
else:
logger.info('git fetch -a')
git.fetch('-a', _cwd=self.checkout_dir)
logger.info('git pull --rebase')
git.pull('--rebase', _cwd=self.checkout_dir)
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name), download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'), redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def repository(namespace, name, branch='master'):
'''Returns a repository'''
with TemporaryDirectory() as download_path:
old_directory = str(pwd()).strip()
try:
git.clone('https://github.com/{0}/{1}.git'.format(namespace, name),
download_path)
cd(download_path)
git.fetch('origin', branch)
git.checkout(branch)
yield (download_path, git('rev-parse', 'HEAD'),
redis.Dict(key="{0}.{1}".format(namespace, name)))
except ErrorReturnCode_128:
mkdir(download_path)
yield (None, None, None)
cd(old_directory)
def merge_and_check(base, head):
"""Merge <head> into <base>, then run some tests.
Only modifies the working tree---doesn't actually create a merge
commit. Resets and cleans the repo and leaves it in headless mode.
Raises sh.ErrorReturnCode if the merge or the tests fail.
"""
# Make sure we're up to date
git.fetch()
# Make sure we can do a clean checkout
git.reset(hard=True)
git.clean('-dfx')
git.checkout('origin/' + base)
# Merge the working tree, but don't modify the index
git.merge('origin/' + head, no_commit=True)
# Check the PR!
check()
def merge_and_check(base, head):
"""Merge <head> into <base>, then run some tests.
Only modifies the working tree---doesn't actually create a merge
commit. Resets and cleans the repo and leaves it in headless mode.
Raises sh.ErrorReturnCode if the merge or the tests fail.
"""
# Make sure we're up to date
git.fetch()
# Make sure we can do a clean checkout
git.reset(hard=True)
git.clean('-dfx')
git.checkout('origin/' + base)
# Merge the working tree, but don't modify the index
git.merge('origin/' + head, no_commit=True)
# Check the PR!
check()
def deploy_code(repo):
logging.info("Deploying repo: %s, back2github: %s", repo["dir"], repo["sync"])
try:
logging.info("Change working directory to %s", repo["dir"])
sh.cd(repo["dir"])
logging.info("Update code, git pull origin")
git.pull("origin", _out=logging.info, _err=logging.error)
logging.info("done")
logging.info("Delete old branches, git fetch -p")
git.fetch("origin", "-p", _out=logging.info, _err=logging.error)
logging.info("done")
if repo["sync"]:
logging.info("Back2github, git push github")
git.push("github", _out=logging.info, _err=logging.error, _bg=True)
except Exception as e:
logging.info("Deploy error: %s", e)
def lookup_branches(self, branch_names):
try:
git.fetch(self.git_remote.name, branch_names)
except:
return None
remote_branches = []
for branch_name in branch_names:
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name,
pygit2.GIT_BRANCH_REMOTE)
# Make another check for the branch being None
# As observed in issue : https://github.com/sdg-mit/gitless/issues/211
if git_branch is None:
git.fetch(self.git_remote.name)
git_branch = self.gl_repo.git_repo.lookup_branch(
self.git_remote.name + '/' + branch_name,
pygit2.GIT_BRANCH_REMOTE)
remote_branches.append(RemoteBranch(git_branch, self.gl_repo))
return remote_branches
def clone(self):
self.tmp_dir = tempfile.mkdtemp(suffix='facio')
try:
from sh import git
except ImportError:
raise Exception # TODO: Custom exception
try:
git = git.bake(_cwd=self.tmp_dir)
git.clone(self.repo, self.tmp_dir)
git.fetch('--all')
git.checkout('master') # TODO: Branch prompt to the user
except Exception:
raise Exception # TODO: Custom exception
rmtree(os.path.join(self.tmp_dir, '.git'))
with indent(4, quote=' >'):
puts(blue('Clone complete'))
def sync(path, git_reference):
new_env = resetEnv()
logger.debug('Syncing {}'.format(path))
old_path = os.getcwd()
jobid = get_current_job().id
_open_console(jobid)
try:
os.chdir(path)
_log_console(jobid, 'Syncing project with Git.\n')
_l = lambda line: _log_console(jobid, str(line))
git.fetch(_out=_l, _err=_l, _env=new_env).wait()
git.reset(
'--hard',
'origin/{}'.format(git_reference),
_out=_l,
_err=_l,
_env=new_env).wait()
git.submodule(
'sync',
_out=_l,
_err=_l,
_env=new_env).wait()
git.submodule(
'update',
_out=_l,
_err=_l,
_env=new_env).wait()
except:
logger.error(
'Failed to sync project at {}'.format(path),
exc_info=True
)
_close_console(jobid)
os.chdir(old_path)
def clone(self):
""" Clone the git repository into a temporary directory. """
try:
from sh import git
except ImportError:
raise FacioException('Git must be installed to use git+ '
'template paths')
temp_diretory = self.get_temp_directory()
self.out('Git Cloning {0} to {1}'.format(self.path, temp_diretory))
try:
git = git.bake(_cwd=temp_diretory)
git.clone(self.path, temp_diretory)
git.fetch('--all')
git.checkout('master')
except:
raise FacioException('Failed to clone git repository '
'at {0}'.format(self.path))
return temp_diretory
def load_code(repo, ref):
print("loading code from " + repo)
os.chdir(cwd)
# Look up our original repo so that we only load objects once.
base_repo = redis.get("source:" + repo)
if base_repo is None:
r = requests.get("https://api.github.com/repos/" + repo,
auth=(config["overall"]["github-username"],
github_personal_access_token))
r = r.json()
base_repo = "source"
if "source" in r:
base_repo = r["source"]["full_name"]
redis.set("source:" + repo, base_repo)
if base_repo is "source":
base_repo = repo
if type(base_repo) is bytes:
base_repo = base_repo.decode("utf-8")
print("Source repo of " + repo + " is " + base_repo)
repo_path = "repos/" + base_repo
github_base_url = "https://github.com/" + base_repo + ".git"
github_head_url = "https://github.com/" + repo + ".git"
print("waiting for repo lock")
with redis.lock(base_repo, timeout=5 * 60, blocking_timeout=20 * 60):
if not os.path.isdir(repo_path):
os.makedirs(repo_path)
git.clone(github_base_url, repo_path)
# We must make .tmp after cloning because cloning will fail when the
# directory isn't empty.
os.makedirs(repo_path + "/.tmp")
os.chdir(repo_path)
git.fetch(github_head_url, ref)
print("loaded", repo, ref)
def main():
github_remote_url = f"{get_github_url()}/{get_input('target_remote')}.git"
work_dir = pathlib.Path(get_input("work_dir"))
if work_dir.is_dir() and len(list(work_dir.iterdir())) > 0:
os.chdir(work_dir)
remote = "origin"
if get_input("source_remote_name") == remote:
remote = remote + "2"
add_or_set_git_remote(remote, github_remote_url)
git.fetch(remote)
git.checkout("-B", get_input("target_branch"),
f"{remote}/{get_input('target_branch')}")
git.reset("--hard", "HEAD")
git.clean("-fdx")
else:
git.clone("--branch", get_input("target_branch"), github_remote_url,
str(work_dir))
os.chdir(work_dir)
if get_input("target_remote") != get_input("source_remote"):
source_remote_name = get_input("source_remote_name")
add_or_set_git_remote(
source_remote_name,
f"{get_github_url()}/{get_input('source_remote')}.git")
git.fetch(source_remote_name)
set_git_author_info(f"GitHub Action {os.environ['GITHUB_ACTION']}",
"action@localhost")
try:
git("cherry-pick", get_input("source_commits"))
print("Source commits were cherry-picked successfully",
file=sys.stderr)
except sh.ErrorReturnCode:
print("Source commits could not be cherry-picked", file=sys.stderr)
raise
def _get_provisioner_repo(self):
# Use the configured git repository, if any
provisioner_git_repo = self.config.get('git_repo')
provisioner_git_revision = self.config.get('git_revision')
git_local_mirror = self._get_mirror_path(provisioner_git_repo)
zabbix_repo = self.global_config.get('zabbix_repo', ZABBIX_REPO)
lock_path = get_lock_path_from_repo(provisioner_git_repo)
log(
"Getting provisioner features from {r}".format(
r=provisioner_git_repo), self._log_file)
try:
output = git("ls-remote", "--exit-code", provisioner_git_repo,
provisioner_git_revision).strip()
log(
"Provisioner repository checked successfuly with output: " +
output, self._log_file)
except sh.ErrorReturnCode:
log(
"Invalid provisioner repository or invalid credentials. Please check your yaml 'config.yml' file",
self._log_file)
raise
try:
git_acquire_lock(lock_path, self._log_file)
# Creates the Provisioner local mirror
if not os.path.exists(git_local_mirror):
log(
"Creating local mirror [{r}] for the first time".format(
r=git_local_mirror), self._log_file)
os.makedirs(git_local_mirror)
os.chdir(git_local_mirror)
git.init(['--bare'])
git.remote(['add', self.name, provisioner_git_repo])
git.remote(['add', 'zabbix', zabbix_repo])
log(
"Fetching local mirror [{r}] remotes".format(
r=git_local_mirror), self._log_file)
os.chdir(git_local_mirror)
git.fetch(['--all'])
finally:
git_release_lock(lock_path, self._log_file)
log(
"Cloning [{r}] repo with local mirror reference".format(
r=provisioner_git_repo), self._log_file)
git.clone([
'--reference', git_local_mirror, provisioner_git_repo, '-b',
provisioner_git_revision, '--single-branch',
self.local_repo_path + '/'
])
if os.path.exists(self.local_repo_path + '/.gitmodules'):
os.chdir(self.local_repo_path)
log("Re-map submodules on local git mirror", self._log_file)
git_remap_submodule(self.local_repo_path, zabbix_repo,
git_local_mirror, self._log_file)
log("Submodule init and update", self._log_file)
git.submodule('init')
git.submodule('update')
def tag(obj, tag_name, remote, yes):
current_branch = get_current_branch()
remote_url = git.remote("get-url", remote).strip()
gh, repo = obj
tag = split_version(tag_name)
tag_name = format_version(tag)
major, minor, fix = tag
with Spinner(f"Checking for milestone for tag {tag_name}"):
tag_milestone = None
for ms in repo.get_milestones(state="all"):
if ms.title == tag_name:
tag_milestone = ms
break
assert tag_milestone is not None, "Did not find milestone for tag"
release_branch_name = f"release/v{major}.{minor:>02}.X"
with Spinner("Refreshing branches"):
git.fetch(all=True, prune=True)
if fix == 0:
# new minor release
with Spinner(f"Checking out and updating {default_branch_name}"):
git.checkout(default_branch_name)
git.pull()
assert not check_branch_exists(
release_branch_name
), "For new minor: release branch CANNOT exist yet"
with Spinner(f"Creating {release_branch_name}"):
git.checkout("-b", release_branch_name)
else:
assert check_branch_exists(
release_branch_name), "For new fix: release brunch MUST exist"
with Spinner(f"Checking out {release_branch_name}"):
git.checkout(release_branch_name)
# we are not on release branch
version_file = Path("version_number")
assert version_file.exists(), "Version number file not found"
current_version_string = version_file.read_text()
print(f"Current version: [bold]{current_version_string}[/bold]")
if fix == 0:
assert current_version_string == "9.9.9", "Unexpected current version string found"
else:
assert current_version_string != f"{major}.{minor}.{fix-1}", "Unexpected current version string found"
version_string = f"{major}.{minor}.{fix}"
with Spinner(
f"Bumping version number in '{version_file}' to '{version_string}'"
):
with version_file.open("w") as fh:
fh.write(version_string)
with Spinner("Comitting"):
git.add(version_file)
git.commit(m=f"Bump version number to {version_string}")
with Spinner(f"Creating tag {tag_name}"):
git.tag(tag_name)
print(
f"I will now: push tag [bold green]{tag_name}[/bold green] and branch [bold green]{release_branch_name}[/bold green] to [bold]{remote_url}[/bold]"
)
if not confirm("Continue?", yes=yes):
raise SystemExit("Aborting")
with Spinner(f"Pushing branch {release_branch_name}"):
git.push("-u", remote, release_branch_name)
with Spinner(f"Pushing tag {tag_name}"):
git.push(remote, tag_name)
def ensure_i18n_remote(self, args: argparse.Namespace) -> None:
k = {'_cwd': args.root}
if b'i18n' not in git.remote(**k).stdout:
git.remote.add('i18n', args.url, **k)
git.fetch('i18n', **k)
def fetch():
print("git fetch")
return git.fetch()
def _update(self):
git.fetch(self.remote_name, self.branch_name)
self.git_branch = self.gl_repo.git_repo.lookup_branch(
self.remote_name + '/' + self.branch_name, pygit2.GIT_BRANCH_REMOTE)
def git_fetch(remote, branch, file_path):
'''Fetch for remote changes'''
p = git.fetch(remote, branch, _cwd=dir_path(file_path), _tty_out=False)
p.wait()
show_msg('Repo updated', 'Info')
def ensure_i18n_remote(self, args):
k = {'_cwd': args.root}
if b'i18n' not in git.remote(**k).stdout:
git.remote.add('i18n', args.url, **k)
git.fetch('i18n', **k)
def _update(self):
git.fetch(self.remote_name, self.branch_name)
self.git_branch = self.gl_repo.git_repo.lookup_branch(
self.remote_name + '/' + self.branch_name,
pygit2.GIT_BRANCH_REMOTE)
def git_fetch(remote, branch, file_path):
'''Fetch for remote changes'''
p = git.fetch(remote, branch, _cwd=dir_path(file_path), _tty_out=False)
p.wait()
show_msg('Repo updated', 'Info')