def merge_branch(branch, squash=False, strategy=None):
cmd = ['git', 'merge', branch]
if squash:
cmd.append('--squash')
if strategy:
cmd.append('--strategy=' + strategy)
silent_run(cmd)
def merge_branch(branch, squash=False, strategy=None):
cmd = ['git', 'merge', branch]
if squash:
cmd.append('--squash')
if strategy:
cmd.append('--strategy=' + strategy)
silent_run(cmd)
def checkout_branch(branch, repo_path=None):
"""
Checks out the branch in the given or current repo. Raises on error.
:param str branch: Branch to checkout. It can be a branch name or remote/branch combo.
if remote is provided, it will always set upstream to :meth:`upstream_remote`
regardless of the downstream remote, as we always want to track downstream changes to the
upstream remote.
:param str repo_path: Path to repo to run checkout in. Defaults to current.
"""
name = branch.split('/')[-1] if '/' in branch else None
cmd = ['git', 'checkout', branch]
if name:
cmd.extend(['-B', name])
silent_run(cmd, cwd=repo_path)
if name:
upstream_branch = '{}/{}'.format(upstream_remote(), name)
if 'remotes/{}'.format(upstream_branch) in all_branches(remotes=True):
silent_run(
'git branch --set-upstream-to {}'.format(upstream_branch))
else:
click.echo(
'FYI Can not change upstream tracking branch to {} as it does not exist'
.format(upstream_branch))
def add_files(files=None):
if files:
files = ' '.join(files)
silent_run('git add --all ' + files)
else:
files = '.'
silent_run('git add --all ' + files, cwd=repo_path())
def remove_branch(branch, raises=False, remote=False, force=False):
""" Removes branch """
run(['git', 'branch', '-D' if force else '-d', branch], raises=raises)
if remote:
silent_run(['git', 'push',
default_remote(), '--delete', branch],
raises=raises)
def add_files(files=None):
if files:
files = ' '.join(files)
silent_run('git add --all ' + files)
else:
files = '.'
silent_run('git add --all ' + files, cwd=repo_path())
def merge_branch(branch, commit=None, squash=False, strategy=None):
cmd = ['git', 'merge', branch]
if squash:
cmd.append('--squash')
if strategy:
cmd.append('--strategy=' + strategy)
if commit is None:
current = current_branch()
message = f"Merge branch {branch} into {current} (using strategy {strategy})"
cmd.append('-m ' + message)
if commit:
cmd.append(commit)
silent_run(cmd)
def push_repo(path=None, force=False, remote=None, branch=None):
push_opts = []
if force:
push_opts.append('--force')
if not remote_tracking_branch(repo=path):
push_opts.append('--set-upstream ' + remote)
elif remote:
push_opts.append(remote)
if branch:
push_opts.append(branch)
silent_run('git push ' + ' '.join(push_opts), cwd=path)
def push_repo(path=None, force=False, remote=None, branch=None):
push_opts = []
if force:
push_opts.append('--force')
if not remote_tracking_branch(repo=path):
push_opts.append('--set-upstream ' + remote)
elif remote:
push_opts.append(remote)
if branch:
push_opts.append(branch)
silent_run('git push ' + ' '.join(push_opts), cwd=path)
def update_repo(path=None, quiet=False):
""" Updates given or current repo to HEAD """
if not remote_tracking_branch(repo=path):
if not quiet:
click.echo(
'Did not update as remote tracking is not setup for branch')
return
branch = current_branch(repo=path)
if not quiet:
click.echo('Updating ' + branch)
remotes = all_remotes(repo=path)
failed_remotes = []
for remote in remotes:
if len(remotes) > 1 and not quiet:
click.echo(' ... from ' + remote)
output, success = silent_run('git pull --ff-only --tags {} {}'.format(
remote, branch),
cwd=path,
return_output=2)
if not success:
error_match = re.search(r'(?:fatal|ERROR): (.+)', output)
error = error_match.group(1) if error_match else output
click.echo(' ... ' + error.strip(' .'))
failed_remotes.append(remote)
if failed_remotes:
raise SCMError('Failed to pull from remote(s): {}'.format(
', '.join(failed_remotes)))
def _all_remotes(repo=None):
""" Returns all remotes. """
remotes_output = silent_run('git remote', cwd=repo, return_output=True)
remotes = []
if remotes_output:
for remote in remotes_output.split('\n'):
if remote:
remote = remote.strip()
remotes.append(remote)
required_remotes = {
DEFAULT_REMOTE: 'Your fork of the upstream repo',
'upstream': 'The upstream repo'
}
if len(remotes) >= 2 and not set(required_remotes).issubset(set(remotes)):
click.echo('Current remotes: {}'.format(' '.join(remotes)))
click.echo(
'Only the following remotes are required -- please set them up accordingly:'
)
for remote in required_remotes:
click.echo(' {}: {}'.format(remote, required_remotes[remote]))
exit(1)
return remotes
def remote_tracking_branch(repo=None):
remote_output = silent_run('git rev-parse --abbrev-ref --symbolic-full-name @{u}', cwd=repo, return_output=True)
if 'no upstream' in remote_output:
return None
else:
return remote_output
def update_repo(path=None, quiet=False):
""" Updates given or current repo to HEAD """
if not remote_tracking_branch(repo=path):
if not quiet:
click.echo('Did not update as remote tracking is not setup for branch')
return
branch = current_branch(repo=path)
if not quiet:
click.echo('Updating ' + branch)
remotes = all_remotes(repo=path)
failed_remotes = []
for remote in remotes:
if len(remotes) > 1 and not quiet:
click.echo(' ... from ' + remote)
output, success = silent_run('git pull --ff-only --tags {} {}'.format(remote, branch), cwd=path, return_output=2)
if not success:
error_match = re.search(r'(?:fatal|ERROR): (.+)', output)
error = error_match.group(1) if error_match else output
click.echo(' ... ' + error.strip(' .'))
failed_remotes.append(remote)
if failed_remotes:
raise SCMError('Failed to pull from remote(s): {}'.format(', '.join(failed_remotes)))
def checkout_product(product_url, checkout_path):
""" Checks out the product from url. Raises on error """
product_url = product_url.strip('/')
prod_name = product_name(product_url)
if os.path.exists(checkout_path):
log.debug('%s is already checked out.', prod_name)
checkout_branch('master', checkout_path)
return update_repo(checkout_path)
if re.match('[\w-]+$', product_url):
try:
logging.getLogger('requests').setLevel(logging.WARN)
response = requests.get(config.checkout.search_api_url,
params={'q': product_url},
timeout=10)
response.raise_for_status()
results = response.json()['items']
if not results:
log.error('No repo matching "%s" found.', product_url)
sys.exit(1)
product_url = results[0]['ssh_url']
click.echo('Using repo url ' + product_url)
except Exception as e:
log.error('Could not find repo for %s using %s due to error: ',
product_url, config.checkout.search_api_url, e)
sys.exit(1)
elif USER_REPO_REFERENCE_RE.match(product_url):
product_url = config.checkout.user_repo_url % product_url
is_origin = not config.checkout.origin_user or config.checkout.origin_user + '/' in product_url
remote_name = DEFAULT_REMOTE if is_origin else UPSTREAM_REMOTE
silent_run(
['git', 'clone', product_url, checkout_path, '--origin', remote_name])
if not is_origin:
origin_url = re.sub(r'(\.com[:/])(\w+)(/)',
r'\1{}\3'.format(config.checkout.origin_user),
product_url)
silent_run(['git', 'remote', 'add', DEFAULT_REMOTE, origin_url],
cwd=checkout_path)
def remote_tracking_branch(repo=None):
remote_output = silent_run(
'git rev-parse --abbrev-ref --symbolic-full-name @{u}',
cwd=repo,
return_output=True)
if 'no upstream' in remote_output:
return None
else:
return remote_output
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate",
cwd=repo,
shell=True)
click.echo('Removing *.pyc files')
silent_run(
"find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;",
cwd=repo,
shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' %
config.clean.remove_all_products_except)
keep_products = expand_product_groups(
config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' %
config.clean.remove_products_older_than_days)
keep_time = time(
) - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status
or 'working tree clean' in status)
and len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(
' - Skipping "%s" as it has changes that may not be committed'
% name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def checkout_product(product_url, checkout_path):
""" Checks out the product from url. Raises on error """
product_url = product_url.strip('/')
prod_name = product_name(product_url)
if os.path.exists(checkout_path):
log.debug('%s is already checked out.', prod_name)
checkout_branch('master', checkout_path)
return update_repo(checkout_path)
if re.match('[\w-]+$', product_url):
try:
logging.getLogger('requests').setLevel(logging.WARN)
response = requests.get(config.checkout.search_api_url, params={'q': product_url}, timeout=10)
response.raise_for_status()
results = response.json()['items']
if not results:
log.error('No repo matching "%s" found.', product_url)
sys.exit(1)
product_url = results[0]['ssh_url']
click.echo('Using repo url ' + product_url)
except Exception as e:
log.error('Could not find repo for %s using %s due to error: ', product_url,
config.checkout.search_api_url, e)
sys.exit(1)
elif USER_REPO_REFERENCE_RE.match(product_url):
product_url = config.checkout.user_repo_url % product_url
is_origin = not config.checkout.origin_user or config.checkout.origin_user + '/' in product_url
remote_name = DEFAULT_REMOTE if is_origin else UPSTREAM_REMOTE
silent_run(['git', 'clone', product_url, checkout_path, '--origin', remote_name])
if not is_origin:
origin_url = re.sub(r'(\.com[:/])(\w+)(/)', r'\1{}\3'.format(config.checkout.origin_user), product_url)
silent_run(['git', 'remote', 'add', DEFAULT_REMOTE, origin_url], cwd=checkout_path)
def all_branches(repo=None, remotes=False, verbose=False):
""" Returns all branches. The first element is the current branch. """
cmd = ['git', 'branch']
if remotes:
cmd.append('--all')
if verbose:
cmd.append('-vv')
branch_output = silent_run(cmd, cwd=repo, return_output=True)
branches = []
remotes = all_remotes(repo=repo)
up_remote = remotes and upstream_remote(repo=repo, remotes=remotes)
def_remote = remotes and default_remote(repo=repo, remotes=remotes)
if branch_output:
for branch in branch_output.split('\n'):
branch = branch.strip()
if branch:
if verbose:
star, detached, local_branch, remote, branch = REMOTE_BRANCH_RE.search(
branch).groups()
if remote and remotes:
# Rightful/tracking remote differs based on parent vs child branch:
# Parent branch = upstream remote
# Child branch = origin remote
rightful_remote = (
remote == up_remote and '@' not in local_branch
or remote == def_remote and '@' in local_branch)
branch = local_branch if rightful_remote else '{}^{}'.format(
local_branch, shortest_id(remote, remotes))
elif detached:
branch = local_branch + '*'
else:
branch = local_branch
if star:
branches.insert(0, branch)
else:
branches.append(branch)
else:
if branch.startswith('*'):
branches.insert(0, branch.strip('* '))
else:
branches.append(branch)
return branches
def checkout_branch(branch, repo_path=None):
"""
Checks out the branch in the given or current repo. Raises on error.
:param str branch: Branch to checkout. It can be a branch name or remote/branch combo.
if remote is provided, it will always set upstream to :meth:`upstream_remote`
regardless of the downstream remote, as we always want to track downstream changes to the
upstream remote.
:param str repo_path: Path to repo to run checkout in. Defaults to current.
"""
name = branch.split('/')[-1] if '/' in branch else None
cmd = ['git', 'checkout', branch]
if name:
cmd.extend(['-B', name])
silent_run(cmd, cwd=repo_path)
if name:
upstream_branch = '{}/{}'.format(upstream_remote(), name)
if 'remotes/{}'.format(upstream_branch) in all_branches(remotes=True):
silent_run('git branch --set-upstream-to {}'.format(upstream_branch))
else:
click.echo('FYI Can not change upstream tracking branch to {} as it does not exist'.format(upstream_branch))
def all_branches(repo=None, remotes=False, verbose=False):
""" Returns all branches. The first element is the current branch. """
cmd = ['git', 'branch']
if remotes:
cmd.append('--all')
if verbose:
cmd.append('-vv')
branch_output = silent_run(cmd, cwd=repo, return_output=True)
branches = []
remotes = all_remotes(repo=repo)
up_remote = remotes and upstream_remote(repo=repo, remotes=remotes)
def_remote = remotes and default_remote(repo=repo, remotes=remotes)
if branch_output:
for branch in branch_output.split('\n'):
branch = branch.strip()
if branch:
if verbose:
star, detached, local_branch, remote, branch = REMOTE_BRANCH_RE.search(branch).groups()
if remote and remotes:
# Rightful/tracking remote differs based on parent vs child branch:
# Parent branch = upstream remote
# Child branch = origin remote
rightful_remote = (remote == up_remote and '@' not in local_branch or
remote == def_remote and '@' in local_branch)
branch = local_branch if rightful_remote else '{}^{}'.format(local_branch, shortest_id(remote, remotes))
elif detached:
branch = local_branch + '*'
else:
branch = local_branch
if star:
branches.insert(0, branch)
else:
branches.append(branch)
else:
if branch.startswith('*'):
branches.insert(0, branch.strip('* '))
else:
branches.append(branch)
return branches
def test_run(capsys):
with in_temp_dir():
assert run('echo hello > hello.txt; echo world >> hello.txt',
shell=True)
out = run('ls', return_output=True)
assert out == 'hello.txt\n'
out = run(['cat', 'hello.txt'], return_output=True)
assert out == 'hello\nworld\n'
with pytest.raises(RunError):
run('blah')
assert not run('blah', raises=False)
assert silent_run('ls -l')
out, _ = capsys.readouterr()
assert out == ''
def run(self):
repo = repo_path()
if repo:
click.echo('Removing build/dist folders')
silent_run("rm -rf build dist docs/_build */activate", cwd=repo, shell=True)
click.echo('Removing *.pyc files')
silent_run("find . -type d \( -path '*/.tox' -o -path '*/mppy-*' \) -prune -o -name *.pyc -exec rm {} \;", cwd=repo, shell=True)
if self.force:
click.echo('Removing untracked/ignored files')
silent_run('git clean -fdx')
else:
path = workspace_path()
click.echo('Cleaning {}'.format(path))
if config.clean.remove_products_older_than_days or config.clean.remove_all_products_except:
keep_time = 0
keep_products = []
if config.clean.remove_all_products_except:
click.echo('Removing all products except: %s' % config.clean.remove_all_products_except)
keep_products = expand_product_groups(config.clean.remove_all_products_except.split())
if config.clean.remove_products_older_than_days:
click.echo('Removing products older than %s days' % config.clean.remove_products_older_than_days)
keep_time = time() - config.clean.remove_products_older_than_days * 86400
removed_products = []
for repo in repos(path):
name = product_name(repo)
modified_time = os.stat(repo).st_mtime
if keep_products and name not in keep_products or keep_time and modified_time < keep_time:
status = stat_repo(repo, return_output=True)
if (not status or 'nothing to commit' in status and
('working directory clean' in status or 'working tree clean' in status) and
len(all_branches(repo)) <= 1):
shutil.rmtree(repo)
removed_products.append(name)
else:
click.echo(' - Skipping "%s" as it has changes that may not be committed' % name)
if removed_products:
click.echo('Removed ' + ', '.join(removed_products))
def _all_remotes(repo=None):
""" Returns all remotes. """
remotes_output = silent_run('git remote', cwd=repo, return_output=True)
remotes = []
if remotes_output:
for remote in remotes_output.split('\n'):
if remote:
remote = remote.strip()
remotes.append(remote)
required_remotes = {
DEFAULT_REMOTE: 'Your fork of the upstream repo',
'upstream': 'The upstream repo'
}
if len(remotes) >= 2 and not set(required_remotes).issubset(set(remotes)):
click.echo('Current remotes: {}'.format(' '.join(remotes)))
click.echo('Only the following remotes are required -- please set them up accordingly:')
for remote in required_remotes:
click.echo(' {}: {}'.format(remote, required_remotes[remote]))
exit(1)
return remotes
def remove_branch(branch, raises=False, remote=False, force=False):
""" Removes branch """
run(['git', 'branch', '-D' if force else '-d', branch], raises=raises)
if remote:
silent_run(['git', 'push', default_remote(), '--delete', branch], raises=raises)
def update_tags(remote, path=None):
silent_run('git fetch --tags {}'.format(remote), cwd=path)
def run(self):
if self.minor and self.major:
log.error('--minor and --major are mutually exclusive, please use only one.')
return
repo_check()
pypirc = LocalConfig('~/.pypirc')
repository = pypirc.get(self.repo, 'repository')
username = pypirc.get(self.repo, 'username')
password = pypirc.get(self.repo, 'password')
repo_title = 'PyPI' if self.repo == 'pypi' else self.repo.title()
if not repository:
log.error('Please add repository / username to [%s] section in ~/.pypirc', self.repo)
sys.exit(1)
if not username:
username = getpass.getuser('{} Username: '******'{} Password: '******'update')
published_version, changes = self.changes_since_last_publish()
if not changes:
click.echo('There are no changes since last publish')
sys.exit(0)
silent_run('rm -rf dist/*', shell=True, cwd=repo_path())
if self.major or self.minor:
new_version, setup_file = self.bump_version(major=self.major, minor=self.minor)
major_minor = 'major' if self.major else 'minor'
self.commander.run('commit', msg=f'Bump {major_minor} version', files=[setup_file], push=2,
skip_style_check=True)
else:
current_version, setup_file = self.get_version()
# Previously, we publish the current version in setup.py, so need to bump it first before
# we can publish a new version.
if published_version == current_version:
new_version, setup_file = self.bump_version()
else:
new_version = current_version
changelog_file = self.update_changelog(new_version, changes, self.minor or self.major)
tox = ToxIni()
envs = [e for e in tox.envlist if e != 'style']
if envs:
env = envs[0]
if len(envs) > 1:
log.debug('Found multiple default envs in tox.ini, will use first one to build: %s', env)
else:
click.echo('Odd, there are no default envs in tox.ini, so we can not build.')
sys.exit(1)
envdir = tox.envdir(env)
python = os.path.join(envdir, 'bin', 'python')
click.echo('Building source/built distribution')
silent_run(f'{python} setup.py sdist bdist_wheel', cwd=repo_path())
click.echo('Uploading to ' + repo_title)
try:
run('twine upload -r "{repo}" -u "{username}" -p "{password}" dist/*'.format(
repo=self.repo,
username=username,
password=password), shell=True, cwd=repo_path(), silent=2)
except Exception:
sys.exit(1)
self.bump_version()
self.commander.run('commit', msg=PUBLISH_VERSION_PREFIX + new_version, push=2, files=[setup_file, changelog_file],
skip_style_check=True)
def rename_branch(branch, new_branch):
silent_run(['git', 'branch', '-m', branch, new_branch])
def create_branch(branch, from_branch=None):
""" Creates a branch from the current branch. Raises on error """
cmd = ['git', 'checkout', '-b', branch]
if from_branch:
cmd.append(from_branch)
silent_run(cmd)
def commit_changes(msg):
""" Commits any modified or new files with given message. Raises on error """
silent_run(['git', 'commit', '-am', msg])
click.echo('Committed change.')
def checkout_files(files, repo_path=None):
""" Checks out the given list of files. Raises on error. """
silent_run(['git', 'checkout'] + files, cwd=repo_path)
def commit_changes(msg):
""" Commits any modified or new files with given message. Raises on error """
silent_run(['git', 'commit', '-am', msg])
click.echo('Committed change.')
def update_branch(repo=None, parent='master'):
silent_run('git rebase {}'.format(parent), cwd=repo)
def create_branch(branch, from_branch=None):
""" Creates a branch from the current branch. Raises on error """
cmd = ['git', 'checkout', '-b', branch]
if from_branch:
cmd.append(from_branch)
silent_run(cmd)
def run(self):
if self.minor and self.major:
log.error('--minor and --major are mutually exclusive, please use only one.')
return
repo_check()
pypirc = LocalConfig('~/.pypirc')
repository = pypirc.get(self.repo, 'repository')
username = pypirc.get(self.repo, 'username')
password = pypirc.get(self.repo, 'password')
repo_title = 'PyPI' if self.repo == 'pypi' else self.repo.title()
if not repository:
log.error('Please add repository / username to [%s] section in ~/.pypirc', self.repo)
sys.exit(1)
if not username:
username = getpass.getuser('{} Username: '******'{} Password: '******'update')
published_version, changes = self.changes_since_last_publish()
if not changes:
click.echo('There are no changes since last publish')
sys.exit(0)
silent_run('rm -rf dist/*', shell=True, cwd=repo_path())
if self.major or self.minor:
new_version, setup_file = self.bump_version(major=self.major, minor=self.minor)
major_minor = 'major' if self.major else 'minor'
self.commander.run('commit', msg=f'Bump {major_minor} version', files=[setup_file], push=2,
skip_style_check=True)
else:
current_version, setup_file = self.get_version()
# Previously, we publish the current version in setup.py, so need to bump it first before
# we can publish a new version.
if published_version == current_version:
new_version, setup_file = self.bump_version()
else:
new_version = current_version
changelog_file = self.update_changelog(new_version, changes, self.minor or self.major)
tox = ToxIni()
envs = [e for e in tox.envlist if e != 'style']
if envs:
env = envs[0]
if len(envs) > 1:
log.debug('Found multiple default envs in tox.ini, will use first one to build: %s', env)
else:
click.echo('Odd, there are no default envs in tox.ini, so we can not build.')
sys.exit(1)
envdir = tox.envdir(env)
python = os.path.join(envdir, 'bin', 'python')
click.echo('Building source/built distribution')
silent_run(f'{python} setup.py sdist bdist_wheel', cwd=repo_path())
click.echo('Uploading to ' + repo_title)
try:
run('twine upload -r "{repo}" -u "{username}" -p "{password}" dist/*'.format(
repo=self.repo,
username=username,
password=password), shell=True, cwd=repo_path(), silent=2)
except Exception:
sys.exit(1)
self.bump_version()
self.commander.run('commit', msg=PUBLISH_VERSION_PREFIX + new_version, push=2,
files=[setup_file, changelog_file],
skip_style_check=True)
def rename_branch(branch, new_branch):
silent_run(['git', 'branch', '-m', branch, new_branch])
def update_branch(repo=None, parent='master'):
silent_run('git rebase {}'.format(parent), cwd=repo)
def checkout_files(files, repo_path=None):
""" Checks out the given list of files. Raises on error. """
silent_run(['git', 'checkout'] + files, cwd=repo_path)
def update_tags(remote, path=None):
silent_run('git fetch --tags {}'.format(remote), cwd=path)
