def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target
element = {'kind': 'import', 'sources': [repo.source_config(ref=ref)]}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Now add a .gitmodules file with an inconsistent submodule,
# we are calling this inconsistent because the file was created
# but `git submodule add` was never called, so there is no reference
# associated to the submodule.
#
repo.add_file(
os.path.join(project, 'inconsistent-submodule', '.gitmodules'))
# Fetch should work, we're not yet at the offending ref
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
# Track will encounter an inconsistent submodule without any ref
result = cli.run(project=project, args=['track', 'target.bst'])
result.assert_success()
# Assert that we are just fine without it, and emit a warning to the user.
assert "Ignoring inconsistent submodule" in result.stderr
def test_gitlfs_off(cli, tmpdir, datafiles, Explicit):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
element = {
'kind':
'import',
'sources': [{
'kind': 'git_tag',
'url': 'https://gitlab.com/buildstream/testing/test-lfs-repo',
'track': 'master',
'ref': '85b163a6252154d93c3f3320e95866b598c07835',
}]
}
if Explicit:
element['sources'][0]['use-lfs'] = False
_yaml.dump(element, os.path.join(project, 'target.bst'))
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project,
args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
with open(os.path.join(checkoutdir, str('cat.bin')), 'r') as fl:
assert "oid sha256:cacc5757993c49e811bef86a7075581e4737226313a41aed524e89739ea82bfb" in fl.read(
)
def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
# Write out our test target
element = {'kind': 'import', 'sources': [repo.source_config(ref=ref)]}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project,
args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert not os.path.exists(
os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
def run(self, *args, project_config=None, **kwargs):
# First load the project.conf and substitute {project_dir}
#
# Save the original project.conf, because we will run more than
# once in the same temp directory
#
project_directory = kwargs['project']
project_filename = os.path.join(project_directory, 'project.conf')
project_backup = os.path.join(project_directory, 'project.conf.backup')
project_load_filename = project_filename
if not os.path.exists(project_backup):
shutil.copy(project_filename, project_backup)
else:
project_load_filename = project_backup
with open(project_load_filename) as f:
config = f.read()
config = config.format(project_dir=project_directory)
if project_config is not None:
# If a custom project configuration dictionary was
# specified, composite it on top of the already
# substituted base project configuration
#
base_config = _yaml.load_data(config)
# In order to leverage _yaml.composite_dict(), both
# dictionaries need to be loaded via _yaml.load_data() first
#
with tempfile.TemporaryDirectory(dir=project_directory) as scratchdir:
temp_project = os.path.join(scratchdir, 'project.conf')
with open(temp_project, 'w') as f:
yaml.safe_dump(project_config, f)
project_config = _yaml.load(temp_project)
_yaml.composite_dict(base_config, project_config)
base_config = _yaml.node_sanitize(base_config)
_yaml.dump(base_config, project_filename)
else:
# Otherwise, just dump it as is
with open(project_filename, 'w') as f:
f.write(config)
return super().run(*args, **kwargs)
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target with a bad ref
element = {'kind': 'import', 'sources': [repo.source_config(ref='5')]}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Assert that fetch raises an error here
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target
gitsource = repo.source_config(ref=None)
gitsource.pop('track')
element = {'kind': 'import', 'sources': [gitsource]}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Track will encounter an inconsistent submodule without any ref
result = cli.run(project=project, args=['show', 'target.bst'])
result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
result.assert_task_error(None, None)
def configured(directory, config=None):
# Ensure we've at least relocated the caches to a temp directory
if not config:
config = {}
if not config.get('sourcedir', False):
config['sourcedir'] = os.path.join(directory, 'sources')
if not config.get('builddir', False):
config['builddir'] = os.path.join(directory, 'build')
if not config.get('artifactdir', False):
config['artifactdir'] = os.path.join(directory, 'artifacts')
if not config.get('logdir', False):
config['logdir'] = os.path.join(directory, 'logs')
# Dump it and yield the filename for test scripts to feed it
# to buildstream as an artument
filename = os.path.join(directory, "buildstream.conf")
_yaml.dump(config, filename)
yield filename
def test_gitlfs_notset(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
element = {
'kind':
'import',
'sources': [{
'kind': 'git_tag',
'url': 'https://gitlab.com/buildstream/testing/test-lfs-repo',
'track': 'master',
'ref': '85b163a6252154d93c3f3320e95866b598c07835'
}]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
with open(os.path.join(project, 'project.conf'), "a+") as fl:
fl.write("""fatal-warnings:
- git_tag:unused-lfs
""")
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, 'git_tag:unused-lfs')
def test_overwrite_rogue_tag_multiple_remotes(cli, tmpdir, datafiles):
"""When using multiple remotes in cache (i.e. when using aliases), we
need to make sure we override tags. This is not allowed to fetch
tags that were present from different origins
"""
project = str(datafiles)
repofiles = os.path.join(str(tmpdir), 'repofiles')
os.makedirs(repofiles, exist_ok=True)
file0 = os.path.join(repofiles, 'file0')
with open(file0, 'w') as f:
f.write('test\n')
repo = create_repo('git', str(tmpdir))
top_commit = repo.create(repofiles)
repodir, reponame = os.path.split(repo.repo)
project_config = _yaml.load(os.path.join(project, 'project.conf'))
project_config['aliases'] = {'repo': 'http://example.com/'}
project_config['mirrors'] = [{
'name': 'middle-earth',
'aliases': {
'repo': ['file://{}/'.format(repodir)]
}
}]
_yaml.dump(_yaml.node_sanitize(project_config),
os.path.join(project, 'project.conf'))
repo.add_annotated_tag('tag', 'tag')
file1 = os.path.join(repofiles, 'file1')
with open(file1, 'w') as f:
f.write('test\n')
ref = repo.add_file(file1)
config = repo.source_config(ref=ref)
del config['track']
config['url'] = 'repo:{}'.format(reponame)
# Write out our test target
element = {
'kind': 'import',
'sources': [config],
}
element_path = os.path.join(project, 'target.bst')
_yaml.dump(element, element_path)
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
repo.checkout(top_commit)
file2 = os.path.join(repofiles, 'file2')
with open(file2, 'w') as f:
f.write('test\n')
new_ref = repo.add_file(file2)
repo.delete_tag('tag')
repo.add_annotated_tag('tag', 'tag')
repo.checkout('master')
otherpath = os.path.join(str(tmpdir), 'other_path')
shutil.copytree(repo.repo, os.path.join(otherpath, 'repo'))
new_repo = create_repo('git', otherpath)
repodir, reponame = os.path.split(repo.repo)
_yaml.dump(_yaml.node_sanitize(project_config),
os.path.join(project, 'project.conf'))
config = repo.source_config(ref=new_ref)
del config['track']
config['url'] = 'repo:{}'.format(reponame)
element = {
'kind': 'import',
'sources': [config],
}
_yaml.dump(element, element_path)
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()