def remote_set(location, repo, remote='origin'):
"""set the remote at location to repo"""
ensure_dir(location)
with sh.pushd(location):
if remote_exists(location, remote):
git.remote('rm', remote)
git.remote('add', remote, repo)
def remove_all_ignores(location):
"""
Remove .gitignore files under a location.
"""
find = sh.Command('find')
with sh.pushd(location):
find('.', '-name', '.gitignore', '-delete')
def add_all(location, message='Update'):
"""Add everything to repo at location as user."""
with sh.pushd(location):
# Initialize repo if it isn't already
if not is_dir(location):
git.init()
git.add('--all')
# None of these values can be unset or empty strings because we use
# them as git envvars below. Unset values and empty strings will
# cause git to shout about ident errors.
host = socket.getfqdn() or 'localhost'
euid = utils.get_username() or 'unknown'
ruid = utils.get_real_username() or 'unknown'
ename = utils.get_user_fullname() or 'Unknown User'
rname = utils.get_real_user_fullname() or 'Unknown User'
os.environ['GIT_COMMITTER_EMAIL'] = '{}@{}'.format(euid, host)
os.environ['GIT_AUTHOR_EMAIL'] = '{}@{}'.format(ruid, host)
os.environ['GIT_COMMITTER_NAME'] = ename
os.environ['GIT_AUTHOR_NAME'] = rname
try:
git.commit('--quiet', '-m', message)
except ErrorReturnCode:
pass # ignore errors
def add_all(location, message='Update'):
"""Add everything to repo at location as user."""
with sh.pushd(location):
# Initialize repo if it isn't already
if not is_dir(location):
git.init()
git.add('--all')
# None of these values can be unset or empty strings because we use
# them as git envvars below. Unset values and empty strings will
# cause git to shout about ident errors.
host = socket.getfqdn() or 'localhost'
euid = utils.get_username() or 'unknown'
ruid = utils.get_real_username() or 'unknown'
ename = utils.get_user_fullname() or 'Unknown User'
rname = utils.get_real_user_fullname() or 'Unknown User'
os.environ['GIT_COMMITTER_EMAIL'] = '{}@{}'.format(euid, host)
os.environ['GIT_AUTHOR_EMAIL'] = '{}@{}'.format(ruid, host)
os.environ['GIT_COMMITTER_NAME'] = ename
os.environ['GIT_AUTHOR_NAME'] = rname
try:
git.commit('--quiet', '-m', message)
except ErrorReturnCode:
pass # ignore errors
def remote_set(location, repo, remote='origin'):
"""set the remote at location to repo"""
ensure_dir(location)
with sh.pushd(location):
if remote_exists(location, remote):
git.remote('rm', remote)
git.remote('add', remote, repo)
def remove_all_ignores(location):
"""
Remove .gitignore files under a location.
"""
find = sh.Command('find')
with sh.pushd(location):
find('.', '-name', '.gitignore', '-delete')
def info(directory):
"""Compute git version information for a given directory that is
compatible with MediaWiki's GitInfo class.
:param directory: Directory to scan for git information
:returns: Dict of information about current repository state
"""
git_dir = resolve_gitdir(directory)
head_file = os.path.join(git_dir, 'HEAD')
with open(head_file, 'r') as headfile:
head = headfile.read().strip()
if head.startswith('ref: '):
head = head[5:]
if head.startswith('refs/heads/'):
branch = head[11:]
elif head.startswith('refs/tags/'):
branch = head[10:]
else:
branch = head
head_sha1 = get_disclosable_head(directory, branch)
if head_sha1:
with sh.pushd(git_dir):
commit_date = git.show('-s', '--format=%ct', head_sha1).strip()
else:
commit_date = ''
# Requires git v1.7.5+
try:
with sh.pushd(git_dir):
remote_url = git('ls-remote', '--get-url').strip()
except ErrorReturnCode:
remote_url = ''
utils.get_logger().info("Unable to find remote URL for %s", git_dir)
return {
'@directory': directory,
'head': head,
'headSHA1': head_sha1,
'headCommitDate': commit_date,
'branch': branch,
'remoteURL': remote_url,
}
def info(directory):
"""Compute git version information for a given directory that is
compatible with MediaWiki's GitInfo class.
:param directory: Directory to scan for git information
:returns: Dict of information about current repository state
"""
git_dir = resolve_gitdir(directory)
head_file = os.path.join(git_dir, 'HEAD')
with open(head_file, 'r') as headfile:
head = headfile.read().strip()
if head.startswith('ref: '):
head = head[5:]
if head.startswith('refs/heads/'):
branch = head[11:]
elif head.startswith('refs/tags/'):
branch = head[10:]
else:
branch = head
head_sha1 = get_disclosable_head(directory, branch)
if head_sha1:
with sh.pushd(git_dir):
commit_date = git.show('-s', '--format=%ct', head_sha1).strip()
else:
commit_date = ''
# Requires git v1.7.5+
try:
with sh.pushd(git_dir):
remote_url = git('ls-remote', '--get-url').strip()
except ErrorReturnCode:
remote_url = ''
utils.get_logger().info("Unable to find remote URL for %s", git_dir)
return {
'@directory': directory,
'head': head,
'headSHA1': head_sha1,
'headCommitDate': commit_date,
'branch': branch,
'remoteURL': remote_url,
}
def checkout(location, rev):
"""Checkout a git repo sha at a location
"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
logger.debug('Checking out rev: %s at location: %s', rev, location)
git.checkout('--force', '--quiet', rev)
def sync_submodules(location):
"""Sync git submodules on target machines"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
logger.debug('Syncing out submodules')
git.submodule('sync', '--recursive')
def test_clean_tags(self):
with sh.pushd(TEMPDIR):
for _ in range(10):
nexttag = git.next_deploy_tag(TEMPDIR)
git.git.tag(nexttag)
git.clean_tags(TEMPDIR, 2)
tags = git.git.tag('--list').splitlines()
assert tags is not None, 'After clean_tags(2), no tags remain'
assert len(tags) == 2, 'There should only be 2 tags rmaining'
def sync_submodules(location):
"""Sync git submodules on target machines"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
logger.debug('Syncing out submodules')
git.submodule('sync', '--recursive')
def test_clean_tags(self):
with sh.pushd(TEMPDIR):
for _ in range(10):
nexttag = git.next_deploy_tag(TEMPDIR)
git.git.tag(nexttag)
git.clean_tags(TEMPDIR, 2)
tags = git.git.tag('--list').splitlines()
assert tags is not None, 'After clean_tags(2), no tags remain'
assert len(tags) == 2, 'There should only be 2 tags rmaining'
def checkout(location, rev):
"""Checkout a git repo sha at a location
"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
logger.debug('Checking out rev: %s at location: %s', rev, location)
git.checkout('--force', '--quiet', rev)
def remap_submodules(location, server):
"""Remap all submodules to deployment server
This function supports remapping submodules available on the deployment
server. Since the remote is a non-bare repo (as of git 1.7.8) all
submodules should be available over http under the remote server
checkout's git dir:
[server]/[repo]/.git/modules/[submodule_path]
:param location: String path to local git checkout containing a
`.gitmodules` file
:param server: String path to remote, non-bare, repo gitdir
"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
gitmodule = os.path.join(location, '.gitmodules')
if not os.path.isfile(gitmodule):
logger.warning(
'Unable to rewrite_submodules: No .gitmodules in %s', location)
return
logger.info('Updating .gitmodule: %s', os.path.dirname(gitmodule))
# ensure we're working with a non-modified .gitmodules file
git.checkout('.gitmodules')
# get .gitmodule info
modules = git.config('--list', '--file', '.gitmodules')
submodules = {}
for line in modules.split('\n'):
if not line.startswith('submodule.'):
continue
module_conf = line.split('=')
module_name = module_conf[0].strip()
if module_name.endswith('.path'):
name = module_name[len('submodule.'):-len('.path')]
submodules[name] = module_conf[1].strip()
with open(gitmodule, 'w') as module:
for submodule_name, submodule_path in submodules.items():
# Since we're using a non-bare http remote, map the submodule
# to the submodule path under $GIT_DIR/modules subdirectory of
# the superproject (git documentation: https://git.io/v4W9F).
remote_path = '{}/modules/{}'.format(server, submodule_name)
module.write('[submodule "{}"]\n'.format(submodule_name))
module.write('\tpath = {}\n'.format(submodule_path))
module.write('\turl = {}\n'.format(remote_path))
sync_submodules(location)
def remap_submodules(location, server):
"""Remap all submodules to new server (tin)
This function supports remapping submodules available on the deployment
server. Since the remote is a non-bare repo (as of git 1.7.8) all
submodules should be available over http under the remote server
checkout's git dir:
[server]/[repo]/.git/modules/[submodule_path]
:param location: String path to local git checkout containing a
`.gitmodules` file
:param server: String path to remote, non-bare, repo gitdir
"""
ensure_dir(location)
logger = utils.get_logger()
with sh.pushd(location):
gitmodule = os.path.join(location, '.gitmodules')
if not os.path.isfile(gitmodule):
logger.warning(
'Unable to rewrite_submodules: No .gitmodules in %s', location)
return
logger.info('Updating .gitmodule: %s', os.path.dirname(gitmodule))
# ensure we're working with a non-modified .gitmodules file
git.checkout('.gitmodules')
# get .gitmodule info
modules = git.config('--list', '--file', '.gitmodules')
submodules = {}
for line in modules.split('\n'):
if not line.startswith('submodule.'):
continue
module_conf = line.split('=')
module_name = module_conf[0].strip()
if module_name.endswith('.path'):
name = module_name[len('submodule.'):-len('.path')]
submodules[name] = module_conf[1].strip()
with open(gitmodule, 'w') as module:
for submodule_name, submodule_path in submodules.items():
# Since we're using a non-bare http remote, map the submodule
# to the submodule path under $GIT_DIR/modules subdirectory of
# the superproject (git documentation: https://git.io/v4W9F).
remote_path = '{}/modules/{}'.format(server, submodule_name)
module.write('[submodule "{}"]\n'.format(submodule_name))
module.write('\tpath = {}\n'.format(submodule_path))
module.write('\turl = {}\n'.format(remote_path))
sync_submodules(location)
def list_submodules(repo):
"""List all of the submodules of a given respository"""
ensure_dir(repo)
submodules = []
with sh.pushd(repo):
res = git.submodule('status')
for line in res.splitlines():
submodules.append(re.sub(r'-[a-f0-9]{40} ', '', line))
return submodules
def list_submodules(repo):
"""List all of the submodules of a given respository"""
ensure_dir(repo)
submodules = []
with sh.pushd(repo):
res = git.submodule('status')
for line in res.splitlines():
submodules.append(re.sub(r'-[a-f0-9]{40} ', '', line))
return submodules
def update_server_info(has_submodules=False, location=os.getcwd(),
logger=None):
"""runs git update-server-info and tags submodules"""
with sh.pushd(location):
logger.debug('Update server info')
git('update-server-info')
if has_submodules:
git.submodule('foreach', '--recursive', 'git update-server-info')
def update_server_info(has_submodules=False,
location=os.getcwd(),
logger=None):
"""runs git update-server-info and tags submodules"""
with sh.pushd(location):
logger.debug('Update server info')
git('update-server-info')
if has_submodules:
git.submodule('foreach', '--recursive', 'git update-server-info')
def next_deploy_tag(location):
"""Calculates the scap/sync/{date}/{n} tag to use for this deployment"""
ensure_dir(location)
with sh.pushd(location):
timestamp = datetime.utcnow()
date = timestamp.strftime('%F')
args = ['--list']
tag_fmt = os.path.join(TAG_PREFIX, '{}', '*')
args.append(tag_fmt.format(date))
seq = len(git.tag(*args).splitlines()) + 1
tag_fmt = os.path.join(TAG_PREFIX, '{0}', '{1:04d}')
return tag_fmt.format(date, seq)
def next_deploy_tag(location):
"""Calculates the scap/sync/{date}/{n} tag to use for this deployment"""
ensure_dir(location)
with sh.pushd(location):
timestamp = datetime.utcnow()
date = timestamp.strftime('%F')
args = ['--list']
tag_fmt = os.path.join(TAG_PREFIX, '{}', '*')
args.append(tag_fmt.format(date))
seq = len(git.tag(*args).splitlines()) + 1
tag_fmt = os.path.join(TAG_PREFIX, '{0}', '{1:04d}')
return tag_fmt.format(date, seq)
def fetch(location,
repo,
reference=None,
dissociate=True,
recurse_submodules=False,
shallow=False,
bare=False,
config=None):
"""Fetch a git repo to a location"""
if config is None:
config = {}
if is_dir(location):
remote_set(location, repo)
with sh.pushd(location):
cmd = append_jobs_arg(['--tags'])
if recurse_submodules:
cmd.append('--recurse-submodules')
else:
cmd.append('--no-recurse-submodules')
git.fetch(*cmd)
for name, value in config.items():
git.config(name, value)
else:
cmd = append_jobs_arg([])
if shallow:
cmd.append('--depth')
cmd.append('1')
if reference is not None and GIT_VERSION[0] > 1:
ensure_dir(reference)
cmd.append('--reference')
cmd.append(reference)
if dissociate:
cmd.append('--dissociate')
if recurse_submodules:
cmd.append('--recurse-submodules')
if shallow:
cmd.append('--shallow-submodules')
if bare:
cmd.append('--bare')
if config:
for name, value in config.items():
cmd.append('--config')
cmd.append('{}={}'.format(name, value))
cmd.append(repo)
cmd.append(location)
git.clone(*cmd)
def fetch(location, repo, reference=None, dissociate=True,
recurse_submodules=False, shallow=False, bare=False, config=None):
"""Fetch a git repo to a location"""
if config is None:
config = {}
if is_dir(location):
remote_set(location, repo)
with sh.pushd(location):
cmd = append_jobs_arg(['--tags'])
if recurse_submodules:
cmd.append('--recurse-submodules')
else:
cmd.append('--no-recurse-submodules')
git.fetch(*cmd)
for name, value in config.items():
git.config(name, value)
else:
cmd = append_jobs_arg([])
if shallow:
cmd.append('--depth')
cmd.append('1')
if reference is not None and GIT_VERSION[0] > 1:
ensure_dir(reference)
cmd.append('--reference')
cmd.append(reference)
if dissociate:
cmd.append('--dissociate')
if recurse_submodules:
cmd.append('--recurse-submodules')
if shallow:
cmd.append('--shallow-submodules')
if bare:
cmd.append('--bare')
if config:
for name, value in config.items():
cmd.append('--config')
cmd.append('{}={}'.format(name, value))
cmd.append(repo)
cmd.append(location)
git.clone(*cmd)
def garbage_collect(location):
"""Clean up a repo."""
ensure_dir(location)
with sh.pushd(location):
git.gc('--quiet', '--auto')
def setUp(self):
git.init(TEMPDIR)
git.default_ignore(TEMPDIR)
with sh.pushd(TEMPDIR):
_TOUCH('testfile')
git.add_all(TEMPDIR, 'first commit')
def setUp(self):
git.init(TEMPDIR)
git.default_ignore(TEMPDIR)
with sh.pushd(TEMPDIR):
_TOUCH('testfile')
git.add_all(TEMPDIR, 'first commit')
def garbage_collect(location):
"""Clean up a repo."""
ensure_dir(location)
with sh.pushd(location):
git.gc('--quiet', '--auto')