def git_mirror_checkout_recursive(git, mirror_dir, checkout_dir, git_url, config, git_ref=None,
git_depth=-1, is_top_level=True):
""" Mirror (and checkout) a Git repository recursively.
It's not possible to use `git submodule` on a bare
repository, so the checkout must be done before we
know which submodules there are.
Worse, submodules can be identified by using either
absolute URLs or relative paths. If relative paths
are used those need to be relocated upon mirroring,
but you could end up with `../../../../blah` and in
that case conda-build could be tricked into writing
to the root of the drive and overwriting the system
folders unless steps are taken to prevent that.
"""
if config.verbose:
stdout = None
stderr = None
else:
FNULL = open(os.devnull, 'w')
stdout = FNULL
stderr = FNULL
if not mirror_dir.startswith(config.git_cache + os.sep):
sys.exit("Error: Attempting to mirror to %s which is outside of GIT_CACHE %s"
% (mirror_dir, config.git_cache))
# This is necessary for Cygwin git and m2-git, although it is fixed in newer MSYS2.
git_mirror_dir = convert_path_for_cygwin_or_msys2(git, mirror_dir)
git_checkout_dir = convert_path_for_cygwin_or_msys2(git, checkout_dir)
if not isdir(os.path.dirname(mirror_dir)):
os.makedirs(os.path.dirname(mirror_dir))
if isdir(mirror_dir):
if git_ref != 'HEAD':
check_call_env([git, 'fetch'], cwd=mirror_dir, stdout=stdout, stderr=stderr)
else:
# Unlike 'git clone', fetch doesn't automatically update the cache's HEAD,
# So here we explicitly store the remote HEAD in the cache's local refs/heads,
# and then explicitly set the cache's HEAD.
# This is important when the git repo is a local path like "git_url: ../",
# but the user is working with a branch other than 'master' without
# explicitly providing git_rev.
check_call_env([git, 'fetch', 'origin', '+HEAD:_conda_cache_origin_head'],
cwd=mirror_dir, stdout=stdout, stderr=stderr)
check_call_env([git, 'symbolic-ref', 'HEAD', 'refs/heads/_conda_cache_origin_head'],
cwd=mirror_dir, stdout=stdout, stderr=stderr)
else:
args = [git, 'clone', '--mirror']
if git_depth > 0:
args += ['--depth', str(git_depth)]
try:
check_call_env(args + [git_url, git_mirror_dir], stdout=stdout, stderr=stderr)
except CalledProcessError:
# on windows, remote URL comes back to us as cygwin or msys format. Python doesn't
# know how to normalize it. Need to convert it to a windows path.
if sys.platform == 'win32' and git_url.startswith('/'):
git_url = convert_unix_path_to_win(git_url)
if os.path.exists(git_url):
# Local filepaths are allowed, but make sure we normalize them
git_url = normpath(git_url)
check_call_env(args + [git_url, git_mirror_dir], stdout=stdout, stderr=stderr)
assert isdir(mirror_dir)
# Now clone from mirror_dir into checkout_dir.
check_call_env([git, 'clone', git_mirror_dir, git_checkout_dir], stdout=stdout, stderr=stderr)
if is_top_level:
checkout = git_ref
if git_url.startswith('.'):
output = check_output_env([git, "rev-parse", checkout], stdout=stdout, stderr=stderr)
checkout = output.decode('utf-8')
if config.verbose:
print('checkout: %r' % checkout)
if checkout:
check_call_env([git, 'checkout', checkout],
cwd=checkout_dir, stdout=stdout, stderr=stderr)
# submodules may have been specified using relative paths.
# Those paths are relative to git_url, and will not exist
# relative to mirror_dir, unless we do some work to make
# it so.
try:
submodules = check_output_env([git, 'config', '--file', '.gitmodules', '--get-regexp',
'url'], stderr=stdout, cwd=checkout_dir)
submodules = submodules.decode('utf-8').splitlines()
except CalledProcessError:
submodules = []
for submodule in submodules:
matches = git_submod_re.match(submodule)
if matches and matches.group(2)[0] == '.':
submod_name = matches.group(1)
submod_rel_path = matches.group(2)
submod_url = urljoin(git_url + '/', submod_rel_path)
submod_mirror_dir = os.path.normpath(
os.path.join(mirror_dir, submod_rel_path))
if config.verbose:
print('Relative submodule %s found: url is %s, submod_mirror_dir is %s' % (
submod_name, submod_url, submod_mirror_dir))
with TemporaryDirectory() as temp_checkout_dir:
git_mirror_checkout_recursive(git, submod_mirror_dir, temp_checkout_dir, submod_url,
config, git_ref, git_depth, False)
if is_top_level:
# Now that all relative-URL-specified submodules are locally mirrored to
# relatively the same place we can go ahead and checkout the submodules.
check_call_env([git, 'submodule', 'update', '--init',
'--recursive'], cwd=checkout_dir, stdout=stdout, stderr=stderr)
git_info(config)
if not config.verbose:
FNULL.close()
def git_mirror_checkout_recursive(git,
mirror_dir,
checkout_dir,
git_url,
git_cache,
git_ref=None,
git_depth=-1,
is_top_level=True,
verbose=True):
""" Mirror (and checkout) a Git repository recursively.
It's not possible to use `git submodule` on a bare
repository, so the checkout must be done before we
know which submodules there are.
Worse, submodules can be identified by using either
absolute URLs or relative paths. If relative paths
are used those need to be relocated upon mirroring,
but you could end up with `../../../../blah` and in
that case conda-build could be tricked into writing
to the root of the drive and overwriting the system
folders unless steps are taken to prevent that.
"""
if verbose:
stdout = None
stderr = None
else:
FNULL = open(os.devnull, 'wb')
stdout = FNULL
stderr = FNULL
if not mirror_dir.startswith(git_cache + os.sep):
sys.exit(
"Error: Attempting to mirror to %s which is outside of GIT_CACHE %s"
% (mirror_dir, git_cache))
# This is necessary for Cygwin git and m2-git, although it is fixed in newer MSYS2.
git_mirror_dir = convert_path_for_cygwin_or_msys2(git,
mirror_dir).rstrip('/')
git_checkout_dir = convert_path_for_cygwin_or_msys2(
git, checkout_dir).rstrip('/')
# Set default here to catch empty dicts
git_ref = git_ref or 'HEAD'
mirror_dir = mirror_dir.rstrip('/')
if not isdir(os.path.dirname(mirror_dir)):
os.makedirs(os.path.dirname(mirror_dir))
if isdir(mirror_dir):
try:
if git_ref != 'HEAD':
check_call_env([git, 'fetch'],
cwd=mirror_dir,
stdout=stdout,
stderr=stderr)
else:
# Unlike 'git clone', fetch doesn't automatically update the cache's HEAD,
# So here we explicitly store the remote HEAD in the cache's local refs/heads,
# and then explicitly set the cache's HEAD.
# This is important when the git repo is a local path like "git_url: ../",
# but the user is working with a branch other than 'master' without
# explicitly providing git_rev.
check_call_env(
[git, 'fetch', 'origin', '+HEAD:_conda_cache_origin_head'],
cwd=mirror_dir,
stdout=stdout,
stderr=stderr)
check_call_env([
git, 'symbolic-ref', 'HEAD',
'refs/heads/_conda_cache_origin_head'
],
cwd=mirror_dir,
stdout=stdout,
stderr=stderr)
except CalledProcessError:
msg = ("Failed to update local git cache. "
"Deleting local cached repo: {} ".format(mirror_dir))
print(msg)
# Maybe the failure was caused by a corrupt mirror directory.
# Delete it so the user can try again.
shutil.rmtree(mirror_dir)
raise
else:
args = [git, 'clone', '--mirror']
if git_depth > 0:
args += ['--depth', str(git_depth)]
try:
check_call_env(args + [git_url, git_mirror_dir],
stdout=stdout,
stderr=stderr)
except CalledProcessError:
# on windows, remote URL comes back to us as cygwin or msys format. Python doesn't
# know how to normalize it. Need to convert it to a windows path.
if sys.platform == 'win32' and git_url.startswith('/'):
git_url = convert_unix_path_to_win(git_url)
if os.path.exists(git_url):
# Local filepaths are allowed, but make sure we normalize them
git_url = normpath(git_url)
check_call_env(args + [git_url, git_mirror_dir],
stdout=stdout,
stderr=stderr)
assert isdir(mirror_dir)
# Now clone from mirror_dir into checkout_dir.
check_call_env([git, 'clone', git_mirror_dir, git_checkout_dir],
stdout=stdout,
stderr=stderr)
if is_top_level:
checkout = git_ref
if git_url.startswith('.'):
output = check_output_env([git, "rev-parse", checkout],
stdout=stdout,
stderr=stderr)
checkout = output.decode('utf-8')
if verbose:
print('checkout: %r' % checkout)
if checkout:
check_call_env([git, 'checkout', checkout],
cwd=checkout_dir,
stdout=stdout,
stderr=stderr)
# submodules may have been specified using relative paths.
# Those paths are relative to git_url, and will not exist
# relative to mirror_dir, unless we do some work to make
# it so.
try:
submodules = check_output_env(
[git, 'config', '--file', '.gitmodules', '--get-regexp', 'url'],
stderr=stdout,
cwd=checkout_dir)
submodules = submodules.decode('utf-8').splitlines()
except CalledProcessError:
submodules = []
for submodule in submodules:
matches = git_submod_re.match(submodule)
if matches and matches.group(2)[0] == '.':
submod_name = matches.group(1)
submod_rel_path = matches.group(2)
submod_url = urljoin(git_url + '/', submod_rel_path)
submod_mirror_dir = os.path.normpath(
os.path.join(mirror_dir, submod_rel_path))
if verbose:
print(
'Relative submodule {} found: url is {}, submod_mirror_dir is {}'
.format(submod_name, submod_url, submod_mirror_dir))
with TemporaryDirectory() as temp_checkout_dir:
git_mirror_checkout_recursive(git,
submod_mirror_dir,
temp_checkout_dir,
submod_url,
git_cache=git_cache,
git_ref=git_ref,
git_depth=git_depth,
is_top_level=False,
verbose=verbose)
if is_top_level:
# Now that all relative-URL-specified submodules are locally mirrored to
# relatively the same place we can go ahead and checkout the submodules.
check_call_env([git, 'submodule', 'update', '--init', '--recursive'],
cwd=checkout_dir,
stdout=stdout,
stderr=stderr)
git_info(checkout_dir, None, git=git, verbose=verbose)
if not verbose:
FNULL.close()
def test_relative_git_url_submodule_clone(testing_workdir, monkeypatch):
"""
A multi-part test encompassing the following checks:
1. That git submodules identified with both relative and absolute URLs can be mirrored
and cloned.
2. That changes pushed to the original repository are updated in the mirror and finally
reflected in the package version and filename via `GIT_DESCRIBE_TAG`.
3. That `source.py` is using `check_call_env` and `check_output_env` and that those
functions are using tools from the build env.
"""
toplevel = os.path.join(testing_workdir, 'toplevel')
os.mkdir(toplevel)
relative_sub = os.path.join(testing_workdir, 'relative_sub')
os.mkdir(relative_sub)
absolute_sub = os.path.join(testing_workdir, 'absolute_sub')
os.mkdir(absolute_sub)
sys_git_env = os.environ.copy()
sys_git_env['GIT_AUTHOR_NAME'] = 'conda-build'
sys_git_env['GIT_AUTHOR_EMAIL'] = '*****@*****.**'
sys_git_env['GIT_COMMITTER_NAME'] = 'conda-build'
sys_git_env['GIT_COMMITTER_EMAIL'] = '*****@*****.**'
# Find the git executable before putting our dummy one on PATH.
git = find_executable('git')
# Put the broken git on os.environ["PATH"]
exename = dummy_executable(testing_workdir, 'git')
monkeypatch.setenv("PATH", testing_workdir, prepend=os.pathsep)
# .. and ensure it gets run (and fails).
FNULL = open(os.devnull, 'w')
# Strangely ..
# stderr=FNULL suppresses the output from echo on OS X whereas
# stdout=FNULL suppresses the output from echo on Windows
with pytest.raises(subprocess.CalledProcessError, message="Dummy git was not executed"):
check_call_env([exename, '--version'], stdout=FNULL, stderr=FNULL)
FNULL.close()
for tag in range(2):
os.chdir(absolute_sub)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('absolute', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'absolute'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'absolute{}'.format(tag)],
env=sys_git_env)
os.chdir(relative_sub)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('relative', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'relative'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'relative{}'.format(tag)],
env=sys_git_env)
os.chdir(toplevel)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('toplevel', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'toplevel'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'toplevel{}'.format(tag)],
env=sys_git_env)
if tag == 0:
check_call_env([git, 'submodule', 'add',
convert_path_for_cygwin_or_msys2(git, absolute_sub), 'absolute'],
env=sys_git_env)
check_call_env([git, 'submodule', 'add', '../relative_sub', 'relative'],
env=sys_git_env)
else:
# Once we use a more recent Git for Windows than 2.6.4 on Windows or m2-git we
# can change this to `git submodule update --recursive`.
check_call_env([git, 'submodule', 'foreach', git, 'pull'], env=sys_git_env)
check_call_env([git, 'commit', '-am', 'added submodules@{}'.format(tag)],
env=sys_git_env)
check_call_env([git, 'tag', '-a', str(tag), '-m', 'tag {}'.format(tag)],
env=sys_git_env)
# It is possible to use `Git for Windows` here too, though you *must* not use a different
# (type of) git than the one used above to add the absolute submodule, because .gitmodules
# stores the absolute path and that is not interchangeable between MSYS2 and native Win32.
#
# Also, git is set to False here because it needs to be rebuilt with the longer prefix. As
# things stand, my _b_env folder for this test contains more than 80 characters.
requirements = ('requirements', OrderedDict([
('build',
['git # [False]',
'm2-git # [win]',
'm2-filesystem # [win]'])]))
filename = os.path.join(testing_workdir, 'meta.yaml')
data = OrderedDict([
('package', OrderedDict([
('name', 'relative_submodules'),
('version', '{{ GIT_DESCRIBE_TAG }}')])),
('source', OrderedDict([
('git_url', toplevel),
('git_tag', str(tag))])),
requirements,
('build', OrderedDict([
('script',
['git --no-pager submodule --quiet foreach git log -n 1 --pretty=format:%%s > '
'%PREFIX%\\summaries.txt # [win]',
'git --no-pager submodule --quiet foreach git log -n 1 --pretty=format:%s > '
'$PREFIX/summaries.txt # [not win]'])
])),
('test', OrderedDict([
('commands',
['echo absolute{}relative{} > %PREFIX%\\expected_summaries.txt # [win]'
.format(tag, tag),
'fc.exe /W %PREFIX%\\expected_summaries.txt %PREFIX%\\summaries.txt # [win]',
'echo absolute{}relative{} > $PREFIX/expected_summaries.txt # [not win]'
.format(tag, tag),
'diff -wuN ${PREFIX}/expected_summaries.txt ${PREFIX}/summaries.txt # [not win]'])
]))
])
with open(filename, 'w') as outfile:
outfile.write(yaml.dump(data, default_flow_style=False, width=999999999))
# Reset the path because our broken, dummy `git` would cause `render_recipe`
# to fail, while no `git` will cause the build_dependencies to be installed.
monkeypatch.undo()
# This will (after one spin round the loop) install and run 'git' with the
# build env prepended to os.environ[]
output = api.get_output_file_path(testing_workdir)[0]
assert ("relative_submodules-{}-".format(tag) in output)
api.build(testing_workdir)
def test_relative_git_url_submodule_clone(testing_workdir, monkeypatch):
"""
A multi-part test encompassing the following checks:
1. That git submodules identified with both relative and absolute URLs can be mirrored
and cloned.
2. That changes pushed to the original repository are updated in the mirror and finally
reflected in the package version and filename via `GIT_DESCRIBE_TAG`.
3. That `source.py` is using `check_call_env` and `check_output_env` and that those
functions are using tools from the build env.
"""
toplevel = os.path.join(testing_workdir, 'toplevel')
os.mkdir(toplevel)
relative_sub = os.path.join(testing_workdir, 'relative_sub')
os.mkdir(relative_sub)
absolute_sub = os.path.join(testing_workdir, 'absolute_sub')
os.mkdir(absolute_sub)
sys_git_env = os.environ.copy()
sys_git_env['GIT_AUTHOR_NAME'] = 'conda-build'
sys_git_env['GIT_AUTHOR_EMAIL'] = '*****@*****.**'
sys_git_env['GIT_COMMITTER_NAME'] = 'conda-build'
sys_git_env['GIT_COMMITTER_EMAIL'] = '*****@*****.**'
# Find the git executable before putting our dummy one on PATH.
git = find_executable('git')
# Put the broken git on os.environ["PATH"]
exename = dummy_executable(testing_workdir, 'git')
monkeypatch.setenv("PATH", testing_workdir, prepend=os.pathsep)
# .. and ensure it gets run (and fails).
FNULL = open(os.devnull, 'w')
# Strangely ..
# stderr=FNULL suppresses the output from echo on OS X whereas
# stdout=FNULL suppresses the output from echo on Windows
with pytest.raises(subprocess.CalledProcessError,
message="Dummy git was not executed"):
check_call_env([exename, '--version'], stdout=FNULL, stderr=FNULL)
FNULL.close()
for tag in range(2):
os.chdir(absolute_sub)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('absolute', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'absolute'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'absolute{}'.format(tag)],
env=sys_git_env)
os.chdir(relative_sub)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('relative', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'relative'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'relative{}'.format(tag)],
env=sys_git_env)
os.chdir(toplevel)
if tag == 0:
check_call_env([git, 'init'], env=sys_git_env)
with open('toplevel', 'w') as f:
f.write(str(tag))
check_call_env([git, 'add', 'toplevel'], env=sys_git_env)
check_call_env([git, 'commit', '-m', 'toplevel{}'.format(tag)],
env=sys_git_env)
if tag == 0:
check_call_env([
git, 'submodule', 'add',
convert_path_for_cygwin_or_msys2(git, absolute_sub), 'absolute'
],
env=sys_git_env)
check_call_env(
[git, 'submodule', 'add', '../relative_sub', 'relative'],
env=sys_git_env)
else:
# Once we use a more recent Git for Windows than 2.6.4 on Windows or m2-git we
# can change this to `git submodule update --recursive`.
check_call_env([git, 'submodule', 'foreach', git, 'pull'],
env=sys_git_env)
check_call_env(
[git, 'commit', '-am', 'added submodules@{}'.format(tag)],
env=sys_git_env)
check_call_env(
[git, 'tag', '-a',
str(tag), '-m', 'tag {}'.format(tag)],
env=sys_git_env)
# It is possible to use `Git for Windows` here too, though you *must* not use a different
# (type of) git than the one used above to add the absolute submodule, because .gitmodules
# stores the absolute path and that is not interchangeable between MSYS2 and native Win32.
#
# Also, git is set to False here because it needs to be rebuilt with the longer prefix. As
# things stand, my _b_env folder for this test contains more than 80 characters.
requirements = ('requirements',
OrderedDict([('build', [
'git # [False]',
'm2-git # [win]', 'm2-filesystem # [win]'
])]))
filename = os.path.join(testing_workdir, 'meta.yaml')
data = OrderedDict([
('package',
OrderedDict([('name', 'relative_submodules'),
('version', '{{ GIT_DESCRIBE_TAG }}')])),
('source',
OrderedDict([('git_url', toplevel),
('git_tag', str(tag))])), requirements,
('build',
OrderedDict([('script', [
'git --no-pager submodule --quiet foreach git log -n 1 --pretty=format:%%s > '
'%PREFIX%\\summaries.txt # [win]',
'git --no-pager submodule --quiet foreach git log -n 1 --pretty=format:%s > '
'$PREFIX/summaries.txt # [not win]'
])])),
('test',
OrderedDict([('commands', [
'echo absolute{}relative{} > %PREFIX%\\expected_summaries.txt # [win]'
.format(tag, tag),
'fc.exe /W %PREFIX%\\expected_summaries.txt %PREFIX%\\summaries.txt # [win]',
'echo absolute{}relative{} > $PREFIX/expected_summaries.txt # [not win]'
.format(tag, tag),
'diff -wuN ${PREFIX}/expected_summaries.txt ${PREFIX}/summaries.txt # [not win]'
])]))
])
with open(filename, 'w') as outfile:
outfile.write(
yaml.dump(data, default_flow_style=False, width=999999999))
# Reset the path because our broken, dummy `git` would cause `render_recipe`
# to fail, while no `git` will cause the build_dependencies to be installed.
monkeypatch.undo()
# This will (after one spin round the loop) install and run 'git' with the
# build env prepended to os.environ[]
output = api.get_output_file_path(testing_workdir)[0]
assert ("relative_submodules-{}-".format(tag) in output)
api.build(testing_workdir)
