def check_merges(config):
merge_conflicts = []
base_config = config
for path, config in base_config.span_configs():
git = get_git(path)
with OriginalBranch(git):
git.checkout(config.trunk)
for branch in config.branches:
git.checkout(branch)
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(path),
trunk=config.trunk,
branch=branch,
),
if not git_check_merge(config.trunk, branch, git=git):
merge_conflicts.append((path, config.trunk, branch))
print "FAIL"
else:
print "ok"
if merge_conflicts:
print "You must fix the following merge conflicts before rebuilding:"
for cwd, trunk, branch in merge_conflicts:
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
trunk=trunk,
)
git = get_git(cwd)
print_merge_details(branch, trunk, git)
exit(1)
else:
print "No merge conflicts"
def rebuild_staging(config):
all_configs = list(config.span_configs())
context_manager = contextlib.nested(
*[OriginalBranch(get_git(path)) for path, _ in all_configs])
with context_manager:
for path, config in all_configs:
git = get_git(path)
git.checkout('-B', config.name, origin(config.trunk), '--no-track')
for branch in config.branches:
if not has_local(git, branch):
branch = origin(branch)
print " [{cwd}] Merging {branch} into {name}".format(
cwd=path, branch=branch, name=config.name)
git.merge(branch, '--no-edit')
if config.submodules:
for submodule in config.submodules:
git.add(submodule)
git.commit('-m', "update submodule refs", '--no-edit',
'--allow-empty')
# stupid safety check
assert config.name != 'master'
print " [{cwd}] Force pushing to origin {name}".format(
cwd=path,
name=config.name,
)
force_push(git, config.name)
def rebuild_staging(config):
all_configs = list(config.span_configs())
context_manager = contextlib.nested(*[OriginalBranch(get_git(path))
for path, _ in all_configs])
with context_manager:
for path, config in all_configs:
git = get_git(path)
git.checkout('-B', config.name, origin(config.trunk), '--no-track')
for branch in config.branches:
if not has_local(git, branch):
branch = origin(branch)
print " [{cwd}] Merging {branch} into {name}".format(
cwd=path,
branch=branch,
name=config.name
)
git.merge(branch, '--no-edit')
if config.submodules:
for submodule in config.submodules:
git.add(submodule)
git.commit('-m', "update submodule refs", '--no-edit',
'--allow-empty')
# stupid safety check
assert config.name != 'master'
print " [{cwd}] Force pushing to origin {name}".format(
cwd=path,
name=config.name,
)
force_push(git, config.name)
def check_merges(config, print_details=True):
merge_conflicts = []
not_found = []
base_config = config
for path, config in base_config.span_configs():
git = get_git(path)
with OriginalBranch(git):
trunk = origin(config.trunk)
git.checkout('-B', config.name, trunk, '--no-track')
for branch in config.branches:
if not has_local(git, branch):
branch = origin(branch)
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(path),
trunk=trunk,
branch=branch,
),
try:
git.checkout(branch)
except sh.ErrorReturnCode_1 as e:
assert ("error: pathspec '%s' did not "
"match any file(s) known to git." %
branch) in e.stderr, e.stderr
not_found.append((path, branch))
print "NOT FOUND"
continue
if not git_check_merge(config.name, branch, git=git):
merge_conflicts.append(
(path, origin(config.trunk), branch))
print "FAIL"
else:
print "ok"
if not_found:
print "You must remove the following branches before rebuilding:"
for cwd, branch in not_found:
print " [{cwd}] {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
)
if merge_conflicts:
print "You must fix the following merge conflicts before rebuilding:"
for cwd, trunk, branch in merge_conflicts:
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
trunk=trunk,
)
git = get_git(cwd)
if print_details:
print_merge_details(branch, trunk, git)
if merge_conflicts or not_found:
exit(1)
else:
print "No merge conflicts"
def check_merges(config, print_details=True):
merge_conflicts = []
not_found = []
base_config = config
for path, config in base_config.span_configs():
git = get_git(path)
with OriginalBranch(git):
trunk = origin(config.trunk)
git.checkout('-B', config.name, trunk, '--no-track')
for branch in config.branches:
if not has_local(git, branch):
branch = origin(branch)
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(path),
trunk=trunk,
branch=branch,
),
try:
git.checkout(branch)
except sh.ErrorReturnCode_1 as e:
assert (
"error: pathspec '%s' did not "
"match any file(s) known to git." % branch) in e.stderr, e.stderr
not_found.append((path, branch))
print "NOT FOUND"
continue
if not git_check_merge(config.name, branch, git=git):
merge_conflicts.append((path, origin(config.trunk), branch))
print "FAIL"
else:
print "ok"
if not_found:
print "You must remove the following branches before rebuilding:"
for cwd, branch in not_found:
print " [{cwd}] {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
)
if merge_conflicts:
print "You must fix the following merge conflicts before rebuilding:"
for cwd, trunk, branch in merge_conflicts:
print " [{cwd}] {trunk} => {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
trunk=trunk,
)
git = get_git(cwd)
if print_details:
print_merge_details(branch, trunk, git)
if merge_conflicts or not_found:
exit(1)
else:
print "No merge conflicts"
def fetch_remote(base_config, name="origin"):
jobs = []
seen = set()
fetched = set()
for path, config in base_config.span_configs():
if path in seen:
continue
seen.add(path)
git = get_git(path)
print(" [{cwd}] fetching {name}".format(cwd=path, name=name))
jobs.append(gevent.spawn(git.fetch, name))
for branch in (b for b in config.branches if ":" in b):
remote, branch = branch.split(":", 1)
if remote not in git.remote().split():
url = remote_url(git, remote)
print(" [{path}] adding remote: {remote} -> {url}"
.format(**locals()))
git.remote("add", remote, url)
print(" [{path}] fetching {remote} {branch}".format(**locals()))
jobs.append(gevent.spawn(git.fetch, remote, branch))
fetched.add(remote)
for pr in config.pull_requests:
print(" [{path}] fetching pull request {pr}".format(**locals()))
pr = 'pull/{pr}/head:enterprise-{pr}'.format(pr=pr)
jobs.append(gevent.spawn(git.fetch, 'origin', pr))
gevent.joinall(jobs)
print("fetched {}".format(", ".join(['origin'] + sorted(fetched))))
def fetch_remote(base_config, name="origin"):
jobs = []
seen = set()
fetched = set()
for path, config in base_config.span_configs():
if path in seen:
continue
seen.add(path)
git = get_git(path)
print(" [{cwd}] fetching {name}".format(cwd=path, name=name))
jobs.append(gevent.spawn(git.fetch, name))
for branch in (b for b in config.branches if ":" in b):
remote, branch = branch.split(":", 1)
if remote not in git.remote().split():
url = remote_url(git, remote)
print(" [{path}] adding remote: {remote} -> {url}".format(
**locals()))
git.remote("add", remote, url)
print(" [{path}] fetching {remote} {branch}".format(**locals()))
jobs.append(gevent.spawn(git.fetch, remote, branch))
fetched.add(remote)
for pr in config.pull_requests:
print(" [{path}] fetching pull request {pr}".format(**locals()))
pr = 'pull/{pr}/head:enterprise-{pr}'.format(pr=pr)
jobs.append(gevent.spawn(git.fetch, 'origin', pr))
gevent.joinall(jobs)
print("fetched {}".format(", ".join(['origin'] + sorted(fetched))))
def fetch_remote(base_config):
jobs = []
for path in set(path for path, _ in base_config.span_configs()):
git = get_git(path)
print " [{cwd}] fetching all".format(cwd=path)
jobs.append(gevent.spawn(git.fetch, '--all'))
gevent.joinall(jobs)
print "All branches fetched"
def fetch_remote(base_config):
jobs = []
for path in set(path for path, _ in base_config.span_configs()):
git = get_git(path)
print " [{cwd}] fetching all".format(cwd=path)
jobs.append(gevent.spawn(git.fetch, '--all'))
gevent.joinall(jobs)
print "All branches fetched"
def get_remote_branches(origin, git=None):
git = git or get_git()
branches = [
line.strip().replace('origin/HEAD -> ', '')[len(origin) + 1:]
for line in sh.grep(git.branch('--remote'), r'^ {}'.format(
origin)).strip().split('\n')
]
return branches
def get_remote_branches(origin, git=None):
git = git or get_git()
branches = [
line.strip().replace('origin/HEAD -> ', '')[len(origin) + 1:]
for line in
sh.grep(
git.branch('--remote'), r'^ {}'.format(origin)
).strip().split('\n')
]
return branches
def get_unmerged_remote_branches(git=None):
git = git or get_git()
try:
lines = sh.grep(
git.branch('--remote', '--no-merged', 'origin/master'),
'^ origin',
).strip().split('\n')
except sh.ErrorReturnCode_1:
lines = []
branches = [line.strip()[len('origin/'):] for line in lines]
return branches
def get_unmerged_remote_branches(git=None):
git = git or get_git()
try:
lines = sh.grep(
git.branch('--remote', '--no-merged', 'origin/master'),
'^ origin',
).strip().split('\n')
except sh.ErrorReturnCode_1:
lines = []
branches = [line.strip()[len('origin/'):] for line in lines]
return branches
def _make_full_config(path):
path_prefix = '{}/'.format(path) if path else ''
git = get_git(path)
with OriginalBranch(git):
branches = get_unmerged_remote_branches(git)
config = BranchConfig(
branches=branches,
submodules={
submodule: _make_full_config(path_prefix + submodule)
for submodule in git_submodules(git)
})
return config
def _make_full_config(path):
path_prefix = '{}/'.format(path) if path else ''
git = get_git(path)
with OriginalBranch(git):
branches = get_unmerged_remote_branches(git)
config = BranchConfig(
branches=branches,
submodules={
submodule: _make_full_config(
path_prefix + submodule
)
for submodule in git_submodules(git)
}
)
return config
def sync_local_copies(config, push=True):
base_config = config
unpushed_branches = []
def _count_commits(compare_spec):
return int(sh.wc(git.log(compare_spec, '--oneline', _piped=True),
'-l'))
for path, config in base_config.span_configs():
git = get_git(path)
with OriginalBranch(git):
for branch in [config.trunk] + config.branches:
if ":" in branch or not has_local(git, branch):
continue
git.checkout(branch)
unpushed = _count_commits('origin/{0}..{0}'.format(branch))
unpulled = _count_commits('{0}..origin/{0}'.format(branch))
if unpulled or unpushed:
print(
" [{cwd}] {branch}: {unpushed} ahead "
"and {unpulled} behind origin").format(
cwd=path,
branch=branch,
unpushed=unpushed,
unpulled=unpulled,
)
else:
print " [{cwd}] {branch}: Everything up-to-date.".format(
cwd=path,
branch=branch,
)
if unpushed:
unpushed_branches.append((path, branch))
elif unpulled:
print " Fastforwarding your branch to origin"
git.merge('--ff-only', origin(branch))
if unpushed_branches and push:
print "The following branches have commits that need to be pushed:"
for path, branch in unpushed_branches:
print " [{cwd}] {branch}".format(cwd=path, branch=branch)
exit(1)
else:
print "All branches up-to-date."
def sync_local_copies(config, push=True):
base_config = config
unpushed_branches = []
def _count_commits(compare_spec):
return int(sh.wc(git.log(compare_spec, '--oneline', _piped=True), '-l'))
for path, config in base_config.span_configs():
git = get_git(path)
with OriginalBranch(git):
for branch in [config.trunk] + config.branches:
if ":" in branch or not has_local(git, branch):
continue
git.checkout(branch)
unpushed = _count_commits('origin/{0}..{0}'.format(branch))
unpulled = _count_commits('{0}..origin/{0}'.format(branch))
if unpulled or unpushed:
print((" [{cwd}] {branch}: {unpushed} ahead "
"and {unpulled} behind origin").format(
cwd=path,
branch=branch,
unpushed=unpushed,
unpulled=unpulled,
))
else:
print(" [{cwd}] {branch}: Everything up-to-date.".format(
cwd=path,
branch=branch,
))
if unpushed:
unpushed_branches.append((path, branch))
elif unpulled:
print(" Fastforwarding your branch to origin")
git.merge('--ff-only', origin(branch))
if unpushed_branches and push:
print("The following branches have commits that need to be pushed:")
for path, branch in unpushed_branches:
print(" [{cwd}] {branch}".format(cwd=path, branch=branch))
exit(1)
else:
print("All branches up-to-date.")
def rebuild_staging(config, print_details=True, push=True):
merge_conflicts = []
not_found = []
all_configs = list(config.span_configs())
with ExitStack() as stack:
for path, _ in all_configs:
stack.enter_context(OriginalBranch(get_git(path)))
for path, config in all_configs:
git = get_git(path)
try:
git.checkout('-B', config.name, origin(config.trunk),
'--no-track')
except Exception:
git.checkout('-B', config.name, config.trunk, '--no-track')
for branch in config.branches:
remote = ":" in branch
if remote or not has_local(git, branch):
if remote:
remote_branch = branch.replace(":", "/", 1)
else:
remote_branch = origin(branch)
if not has_remote(git, remote_branch):
not_found.append((path, branch))
print(" [{cwd}] {branch} NOT FOUND".format(
cwd=format_cwd(path),
branch=branch,
))
continue
branch = remote_branch
print(" [{cwd}] Merging {branch} into {name}".format(
cwd=path, branch=branch, name=config.name),
end=' ')
try:
git.merge(branch, '--no-edit')
except sh.ErrorReturnCode_1:
merge_conflicts.append((path, branch, config))
try:
git.merge("--abort")
except sh.ErrorReturnCode_128:
pass
print("FAIL")
else:
print("ok")
for pr in config.pull_requests:
branch = "enterprise-{pr}".format(pr=pr)
print(" [{cwd}] Merging {pr} into {name}".format(
cwd=path, pr=pr, name=config.name),
end=' ')
try:
git.merge(branch, '--no-edit')
except sh.ErrorReturnCode_1:
merge_conflicts.append((path, branch, config))
try:
git.merge("--abort")
except sh.ErrorReturnCode_128:
pass
print("FAIL")
else:
print("ok")
if config.submodules:
for submodule in config.submodules:
git.add(submodule)
git.commit('-m', "update submodule refs", '--no-edit',
'--allow-empty')
if push and not (merge_conflicts or not_found):
for path, config in all_configs:
# stupid safety check
assert config.name != 'master', path
print(" [{cwd}] Force pushing to origin {name}".format(
cwd=path,
name=config.name,
))
force_push(get_git(path), config.name)
if not_found:
print("You must remove the following branches before rebuilding:")
for cwd, branch in not_found:
print(" [{cwd}] {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
))
if merge_conflicts:
print("You must fix the following merge conflicts before rebuilding:")
for cwd, branch, config in merge_conflicts:
print("\n[{cwd}] {branch} => {name}".format(
cwd=format_cwd(cwd),
branch=branch,
name=config.name,
))
git = get_git(cwd)
if print_details:
print_conflicts(branch, config, git)
if merge_conflicts or not_found:
exit(1)
def rebuild_staging(config, print_details=True, push=True):
merge_conflicts = []
not_found = []
all_configs = list(config.span_configs())
context_manager = contextlib.nested(*[OriginalBranch(get_git(path))
for path, _ in all_configs])
with context_manager:
for path, config in all_configs:
git = get_git(path)
try:
git.checkout('-B', config.name, origin(config.trunk), '--no-track')
except Exception:
git.checkout('-B', config.name, config.trunk, '--no-track')
for branch in config.branches:
remote = ":" in branch
if remote or not has_local(git, branch):
if remote:
remote_branch = branch.replace(":", "/", 1)
else:
remote_branch = origin(branch)
if not has_remote(git, remote_branch):
not_found.append((path, branch))
print(" [{cwd}] {branch} NOT FOUND".format(
cwd=format_cwd(path),
branch=branch,
))
continue
branch = remote_branch
print(" [{cwd}] Merging {branch} into {name}".format(
cwd=path,
branch=branch,
name=config.name
), end=' ')
try:
git.merge(branch, '--no-edit')
except sh.ErrorReturnCode_1:
merge_conflicts.append((path, branch, config))
try:
git.merge("--abort")
except sh.ErrorReturnCode_128:
pass
print("FAIL")
else:
print("ok")
for pr in config.pull_requests:
branch = "enterprise-{pr}".format(pr=pr)
print(" [{cwd}] Merging {pr} into {name}".format(
cwd=path,
pr=pr,
name=config.name
), end=' ')
try:
git.merge(branch, '--no-edit')
except sh.ErrorReturnCode_1:
merge_conflicts.append((path, branch, config))
try:
git.merge("--abort")
except sh.ErrorReturnCode_128:
pass
print("FAIL")
else:
print("ok")
if config.submodules:
for submodule in config.submodules:
git.add(submodule)
git.commit('-m', "update submodule refs", '--no-edit',
'--allow-empty')
if push and not (merge_conflicts or not_found):
for path, config in all_configs:
# stupid safety check
assert config.name != 'master', path
print(" [{cwd}] Force pushing to origin {name}".format(
cwd=path,
name=config.name,
))
force_push(get_git(path), config.name)
if not_found:
print("You must remove the following branches before rebuilding:")
for cwd, branch in not_found:
print(" [{cwd}] {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
))
if merge_conflicts:
print("You must fix the following merge conflicts before rebuilding:")
for cwd, branch, config in merge_conflicts:
print("\n[{cwd}] {branch} => {name}".format(
cwd=format_cwd(cwd),
branch=branch,
name=config.name,
))
git = get_git(cwd)
if print_details:
print_conflicts(branch, config, git)
if merge_conflicts or not_found:
exit(1)
def main():
import argparse
import yaml
parser = argparse.ArgumentParser(
description='Rebuild the deploy branch for an environment')
parser.add_argument("env", help="Name of the environment")
parser.add_argument("actions", nargs="*")
parser.add_argument("--commcare-hq-root",
help="Path to cloned commcare-hq repository",
default=os.environ.get("COMMCARE_HQ_ROOT"))
parser.add_argument("-v", "--verbose")
parser.add_argument(
"--no-push",
action="store_true",
help="Do not push the changes to remote git repository.")
args = parser.parse_args()
if not args.commcare_hq_root:
print(
red("Path to commcare-hq repository must be provided.\n"
"Use '--commcare-hq-root=[path]' or set the 'COMMCARE_HQ_ROOT' environment variable."
))
exit(1)
config_path = os.path.join("environments", args.env, "deploy_branches.yml")
git = get_git()
print("Fetching master")
git.fetch("origin", "master")
if not args.no_push:
print("Checking branch config for modifications")
if git.diff("origin/master", "--", config_path):
print(
red("'{}' on this branch different from the one on master".
format(config_path)))
exit(1)
with open(config_path) as config_yaml:
config = yaml.safe_load(config_yaml)
if "trunk" in config:
config = BranchConfig.wrap(config)
config.normalize()
repositories = {"dimagi/commcare-hq": config}
elif "dimagi/commcare-hq" in config:
repositories = {
repo: BranchConfig.wrap(repo_config)
for repo, repo_config in config.items()
}
for repo, repo_config in repositories.items():
repo_config.normalize()
if repo == "dimagi/commcare-hq":
repo_config.root = os.path.abspath(args.commcare_hq_root)
else:
env_var = "{}_ROOT".format(re.sub("[/-]", "_", repo).upper())
code_root = os.environ.get(env_var)
if not code_root:
code_root = raw_input(
"Please supply the location of the '{}' repo: ".format(
repo))
if not code_root or not os.path.exists(code_root):
print(
red("Repo path must be supplied. "
"Consider setting the '{}' environment variable".
format(env_var)))
exit(1)
repo_config.root = os.path.abspath(code_root)
else:
print(red("Unexpected format for config file."))
exit(1)
for config in repositories.values():
if not config.check_trunk_is_recent(config.root):
print("The trunk is not based on a very recent commit")
print("Consider using one of the following:")
print(git_recent_tags(config.root))
exit(1)
if not args.actions:
args.actions = 'fetch sync rebuild'.split()
push = not args.no_push
with DisableGitHooks(), ShVerbose(args.verbose):
for repo, config in repositories.items():
print("\nRebuilding '{}' branch in '{}' repo.".format(
config.name, repo))
if 'fetch' in args.actions:
fetch_remote(config)
if 'sync' in args.actions:
sync_local_copies(config, push=push)
if 'rebuild' in args.actions:
rebuild_staging(config, push=push)
def rebuild_staging(config, print_details=True, push=True):
merge_conflicts = []
not_found = []
all_configs = list(config.span_configs())
context_manager = contextlib.nested(
*[OriginalBranch(get_git(path)) for path, _ in all_configs])
with context_manager:
for path, config in all_configs:
git = get_git(path)
git.checkout('-B', config.name, origin(config.trunk), '--no-track')
for branch in config.branches:
remote = ":" in branch
if remote or not has_local(git, branch):
if remote:
remote_branch = branch.replace(":", "/", 1)
else:
remote_branch = origin(branch)
if not has_remote(git, remote_branch):
not_found.append((path, branch))
print " [{cwd}] {branch} NOT FOUND".format(
cwd=format_cwd(path),
branch=branch,
)
continue
branch = remote_branch
print " [{cwd}] Merging {branch} into {name}".format(
cwd=path, branch=branch, name=config.name),
try:
git.merge(branch, '--no-edit')
except sh.ErrorReturnCode_1:
merge_conflicts.append((path, branch, config.name))
try:
git.merge("--abort")
except sh.ErrorReturnCode_128:
pass
print "FAIL"
else:
print "ok"
if config.submodules:
for submodule in config.submodules:
git.add(submodule)
git.commit('-m', "update submodule refs", '--no-edit',
'--allow-empty')
# stupid safety check
assert config.name != 'master'
if push:
print " [{cwd}] Force pushing to origin {name}".format(
cwd=path,
name=config.name,
)
force_push(git, config.name)
if not_found:
print "You must remove the following branches before rebuilding:"
for cwd, branch in not_found:
print " [{cwd}] {branch}".format(
cwd=format_cwd(cwd),
branch=branch,
)
if merge_conflicts:
print "You must fix the following merge conflicts before rebuilding:"
for cwd, branch, name in merge_conflicts:
print " [{cwd}] {branch} => {name}".format(
cwd=format_cwd(cwd),
branch=branch,
name=name,
)
git = get_git(cwd)
if print_details:
print_merge_details(branch, name, git)
if merge_conflicts or not_found:
exit(1)
