def test_run_build_src_package(self):
"""Test run_build, source and backup packages built."""
self.args.build_source_packages = True
# We need to create dummy source trees, but they do not
# actually need to come from checking out the given version
# control locations.
srcdir = os.path.join(self.tempdir, 'src')
create_files(
srcdir, ['build_src_open-123', 'build_src_closed-456'], {
'build_src_open-123/x': 'x',
'build_src_closed-456/y': 'y',
'build_src_closed-456/.git': 'ignore'
}, {})
self.setup_rc('cfg.add_component("build_src_open")\n'
'cfg.build_src_open.version.set("123")\n'
'cfg.build_src_open.vc.set(TarVC("/dummy"))\n'
'cfg.add_component("build_src_closed")\n'
'cfg.build_src_closed.version.set("456")\n'
'cfg.build_src_closed.vc.set(GitVC("/dummy"))\n'
'cfg.source_date_epoch.set(1111199990)\n')
with self.redirect_stdout_stderr():
self.build_context.run_build()
pkg_src = self.relcfg.pkgdir_path(None, '.src.tar.xz')
pkg_backup = self.relcfg.pkgdir_path(None, '.backup.tar.xz')
dir_src = os.path.join(self.tempdir,
'toolchain-1.0-1-aarch64-linux-gnu')
dir_backup = os.path.join(self.tempdir,
'toolchain-1.0-1-aarch64-linux-gnu.backup')
subprocess.run(['tar', '-x', '-f', pkg_src],
cwd=self.tempdir,
check=True)
self.assertEqual(os.listdir(dir_src), ['build_src_open-1.0-1.tar.xz'])
subprocess.run(['tar', '-x', '-f', pkg_backup],
cwd=self.tempdir,
check=True)
self.assertEqual(os.listdir(dir_backup),
['build_src_closed-1.0-1.tar.xz'])
tar_open = os.path.join(dir_src, 'build_src_open-1.0-1.tar.xz')
tar_closed = os.path.join(dir_backup, 'build_src_closed-1.0-1.tar.xz')
self.assertEqual(os.stat(tar_open).st_mtime, 1111199990)
self.assertEqual(os.stat(tar_closed).st_mtime, 1111199990)
subprocess.run(['tar', '-x', '-f', tar_open],
cwd=self.tempdir,
check=True)
subprocess.run(['tar', '-x', '-f', tar_closed],
cwd=self.tempdir,
check=True)
dir_open = os.path.join(self.tempdir, 'build_src_open-1.0-1')
dir_closed = os.path.join(self.tempdir, 'build_src_closed-1.0-1')
self.assertEqual(read_files(dir_open), (set(), {'x': 'x'}, {}))
self.assertEqual(read_files(dir_closed), (set(), {'y': 'y'}, {}))
self.assertEqual(
os.stat(os.path.join(dir_open, 'x')).st_mtime, 1111199990)
self.assertEqual(
os.stat(os.path.join(dir_closed, 'y')).st_mtime, 1111199990)
def test_move_sysroot_executables(self):
"""Test move_sysroot_executables."""
loader = ReleaseConfigTextLoader()
relcfg_text = ('cfg.build.set("x86_64-linux-gnu")\n'
'cfg.target.set("aarch64-linux-gnu")\n'
'cfg.add_component("generic")\n'
'cfg.generic.vc.set(GitVC("dummy"))\n'
'cfg.generic.version.set("1.23")\n'
'cfg.add_component("sysrooted_libc")\n'
'cfg.sysrooted_libc.vc.set(GitVC("dummy"))\n'
'cfg.sysrooted_libc.version.set("1.23")\n'
'cfg.multilibs.set((Multilib("generic", '
'"sysrooted_libc", ()), Multilib("generic", '
'"sysrooted_libc", ("-m64",), '
'sysroot_osdir="../lib64"), Multilib("generic", '
'"sysrooted_libc", ("-mfoo",), '
'sysroot_suffix="foo")))\n')
relcfg = ReleaseConfig(self.context, relcfg_text, loader, self.args)
multilibs = relcfg.multilibs.get()
create_files(self.indir, ['bin1', 'bin2'], {
'bin1/a': 'file bin1/a',
'bin2/b': 'file bin2/b'
}, {})
tree = FSTreeCopy(self.context, self.indir, {'name'})
tree_moved = multilibs[0].move_sysroot_executables(
tree, ('bin1', 'bin2'))
tree_moved.export(self.outdir)
self.assertEqual(read_files(self.outdir),
({'bin1', 'bin2', 'usr', 'usr/lib', 'usr/lib/bin'}, {
'usr/lib/bin/a': 'file bin1/a',
'usr/lib/bin/b': 'file bin2/b'
}, {}))
shutil.rmtree(self.outdir)
tree_moved = multilibs[1].move_sysroot_executables(
tree, ['bin1', 'bin2'])
tree_moved.export(self.outdir)
self.assertEqual(
read_files(self.outdir),
({'bin1', 'bin2', 'usr', 'usr/lib64', 'usr/lib64/bin'}, {
'usr/lib64/bin/a': 'file bin1/a',
'usr/lib64/bin/b': 'file bin2/b'
}, {}))
shutil.rmtree(self.outdir)
# When only one multilib uses the sysroot, the files are kept
# in their original locations as well as being copied.
tree_moved = multilibs[2].move_sysroot_executables(
tree, ('bin1', 'bin2'))
tree_moved.export(self.outdir)
self.assertEqual(read_files(self.outdir),
({'bin1', 'bin2', 'usr', 'usr/lib', 'usr/lib/bin'}, {
'bin1/a': 'file bin1/a',
'bin2/b': 'file bin2/b',
'usr/lib/bin/a': 'file bin1/a',
'usr/lib/bin/b': 'file bin2/b'
}, {}))
def test_svn_copy_without_metadata(self):
"""Test copying checkouts from SVN."""
subprocess.run(['svnadmin', 'create', self.svndir], check=True)
svn_uri = 'file://%s' % self.svndir
subprocess.run(['svn', '-q', 'co', svn_uri, self.codir], check=True)
self.co_file_write('svnfile', 'svnfile contents')
os.mkdir(self.co_file('dir'))
self.co_file_write('dir/file2', 'file2 contents')
subprocess.run(['svn', '-q', 'add', 'svnfile', 'dir'],
cwd=self.codir,
check=True)
subprocess.run(['svn', '-q', 'commit', '-m', 'commit message'],
cwd=self.codir,
check=True)
vc_obj = SvnVC(self.context, svn_uri)
vc_obj.vc_checkout(self.srcdir, False)
os.chmod(os.path.join(self.srcdir, 'svnfile'), stat.S_IRWXU)
vc_obj.copy_without_metadata(self.srcdir, self.srcdir_copy)
self.assertEqual(read_files(self.srcdir_copy),
({'dir'}, {
'svnfile': 'svnfile contents',
'dir/file2': 'file2 contents'
}, {}))
mode = stat.S_IMODE(
os.stat(os.path.join(self.srcdir_copy, 'svnfile')).st_mode)
self.assertEqual(mode, (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH))
def test_fix_perms(self):
"""Test the fix_perms function."""
create_files(self.indir, ['a', 'b', 'b/c'],
{'x': 'file x', 'b/c/y': 'file b/c/y'},
{'dead-symlink': 'bad', 'ext-symlink': '/'})
os.chmod(self.indir, stat.S_IRWXU)
os.chmod(os.path.join(self.indir, 'x'), stat.S_IRWXU | stat.S_IROTH)
os.chmod(os.path.join(self.indir, 'b/c/y'), 0)
fix_perms(self.indir)
self.assertEqual(read_files(self.indir),
({'a', 'b', 'b/c'},
{'x': 'file x', 'b/c/y': 'file b/c/y'},
{'dead-symlink': 'bad', 'ext-symlink': '/'}))
mode_ex = (stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
mode_noex = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
mode = stat.S_IMODE(os.stat(self.indir).st_mode)
self.assertEqual(mode, mode_ex)
mode = stat.S_IMODE(os.stat(os.path.join(self.indir, 'a')).st_mode)
self.assertEqual(mode, mode_ex)
mode = stat.S_IMODE(os.stat(os.path.join(self.indir, 'b')).st_mode)
self.assertEqual(mode, mode_ex)
mode = stat.S_IMODE(os.stat(os.path.join(self.indir, 'b/c')).st_mode)
self.assertEqual(mode, mode_ex)
mode = stat.S_IMODE(os.stat(os.path.join(self.indir, 'x')).st_mode)
self.assertEqual(mode, mode_ex)
mode = stat.S_IMODE(os.stat(os.path.join(self.indir, 'b/c/y')).st_mode)
self.assertEqual(mode, mode_noex)
def test_tar_command_run(self):
"""Test running the command from the tar_command function."""
create_files(self.indir, ['a', 'b', 'b/c'],
{'a1': 'a', 'a2': 'a', 'b/c/a3': 'a', 'b/a4': 'a',
'b1': 'b', 'b/b2': 'b', 'c': 'c'},
{'a-link': 'a1', 'dead-link': 'bad'})
hard_link_files(self.context, self.indir)
test_tar_xz = os.path.join(self.tempdir, 'test.tar.xz')
subprocess.run(tar_command(test_tar_xz, 'top+dir-1.0', 1234567890),
cwd=self.indir, check=True)
subprocess.run(['tar', '-x', '-f', test_tar_xz], cwd=self.tempdir,
check=True)
outdir = os.path.join(self.tempdir, 'top+dir-1.0')
self.assertEqual(read_files(outdir),
({'a', 'b', 'b/c'},
{'a1': 'a', 'a2': 'a', 'b/c/a3': 'a', 'b/a4': 'a',
'b1': 'b', 'b/b2': 'b', 'c': 'c'},
{'a-link': 'a1', 'dead-link': 'bad'}))
stat_a1 = os.stat(os.path.join(outdir, 'a1'))
self.assertEqual(stat_a1.st_nlink, 4)
self.assertEqual(stat_a1.st_mtime, 1234567890)
stat_dead_link = os.stat(os.path.join(outdir, 'dead-link'),
follow_symlinks=False)
self.assertEqual(stat_dead_link.st_mtime, 1234567890)
# Test that the files are correctly sorted in the tarball.
subprocess.run(['xz', '-d', test_tar_xz], check=True)
tarfile_obj = tarfile.open(os.path.join(self.tempdir, 'test.tar'),
'r:')
self.assertEqual(tarfile_obj.getnames(),
['top+dir-1.0', 'top+dir-1.0/a', 'top+dir-1.0/a-link',
'top+dir-1.0/a1', 'top+dir-1.0/a2', 'top+dir-1.0/b',
'top+dir-1.0/b/a4', 'top+dir-1.0/b/b2',
'top+dir-1.0/b/c', 'top+dir-1.0/b/c/a3',
'top+dir-1.0/b1', 'top+dir-1.0/c',
'top+dir-1.0/dead-link'])
def test_run_build_install_tree(self):
"""Test run_build, implicit install tree creation."""
self.setup_rc('cfg.add_component("build_install_tree")\n')
with self.redirect_stdout_stderr():
self.build_context.run_build()
hosts = self.relcfg.hosts.get()
host_b0 = hosts[0].build_cfg
instdir_def = self.relcfg.install_tree_path(host_b0, 'impl-def')
instdir_empty = self.relcfg.install_tree_path(host_b0, 'impl-empty')
instdir_one = self.relcfg.install_tree_path(host_b0, 'impl-one')
instdir_two = self.relcfg.install_tree_path(host_b0, 'impl-two')
self.assertEqual(read_files(instdir_def), ({'q'}, {'q/a': 'a\n'}, {}))
self.assertEqual(read_files(instdir_empty), (set(), {}, {}))
self.assertEqual(read_files(instdir_one), (set(), {'b': 'b\n'}, {}))
self.assertEqual(read_files(instdir_two), (set(), {
'b': 'b\n',
'c': 'c\n'
}, {}))
def test_run_build_package(self):
"""Test run_build, nonempty packages built."""
self.setup_rc('cfg.add_component("build_package")\n'
'cfg.source_date_epoch.set(1111199990)\n')
with self.redirect_stdout_stderr():
self.build_context.run_build()
hosts = self.relcfg.hosts.get()
pkg_0 = self.relcfg.pkgdir_path(hosts[0], '.tar.xz')
pkg_1 = self.relcfg.pkgdir_path(hosts[1], '.tar.xz')
dir_out = os.path.join(self.tempdir, 'toolchain-1.0')
subprocess.run(['tar', '-x', '-f', pkg_0],
cwd=self.tempdir,
check=True)
self.assertEqual(read_files(dir_out), (set(), {
'a1': 'a\n',
'a2': 'a\n',
'a3': 'a\n',
'b': 'b\n'
}, {
'c': 'b'
}))
stat_a1 = os.stat(os.path.join(dir_out, 'a1'))
self.assertEqual(stat_a1.st_nlink, 3)
self.assertEqual(stat_a1.st_mtime, 1111199990)
stat_b = os.stat(os.path.join(dir_out, 'b'))
self.assertEqual(stat_b.st_nlink, 1)
self.assertEqual(stat_b.st_mtime, 1111199990)
shutil.rmtree(dir_out)
subprocess.run(['tar', '-x', '-f', pkg_1],
cwd=self.tempdir,
check=True)
self.assertEqual(read_files(dir_out), (set(), {
'a1': 'a\n',
'a2': 'a\n',
'a3': 'a\n',
'b': 'b\n',
'c': 'b\n'
}, {}))
stat_a1 = os.stat(os.path.join(dir_out, 'a1'))
self.assertEqual(stat_a1.st_nlink, 3)
self.assertEqual(stat_a1.st_mtime, 1111199990)
stat_b = os.stat(os.path.join(dir_out, 'b'))
self.assertEqual(stat_b.st_nlink, 2)
self.assertEqual(stat_b.st_mtime, 1111199990)
def test_replace_symlinks(self):
"""Test the replace_symlinks function."""
create_files(self.indir, ['d1', 'd2'],
{'f': 'f', 'd1/f': 'd1/f', 'd2/f': 'd2/f'},
{'d1/link': '../f', 'd1/d2': '../d2', 'd2/link': '../f'})
replace_symlinks(self.context, self.indir)
self.assertEqual(read_files(self.indir),
({'d1', 'd2', 'd1/d2'},
{'f': 'f', 'd1/f': 'd1/f', 'd2/f': 'd2/f',
'd1/link': 'f', 'd2/link': 'f', 'd1/d2/f': 'd2/f',
'd1/d2/link': 'f'},
{}))
def test_tar_copy_without_metadata(self):
"""Test copying checkouts from tarballs."""
os.mkdir(self.codir)
self.co_file_write('.git', '.git contents')
self.co_file_write('.svn', '.svn contents')
subprocess.run(['tar', '-c', '-z', '-f', 'test.tar.gz', 'co'],
cwd=self.tempdir,
check=True)
vc_obj = TarVC(self.context, os.path.join(self.tempdir, 'test.tar.gz'))
vc_obj.vc_checkout(self.srcdir, False)
vc_obj.copy_without_metadata(self.srcdir, self.srcdir_copy)
self.assertEqual(read_files(self.srcdir_copy), (set(), {
'.git': '.git contents',
'.svn': '.svn contents'
}, {}))
def test_git_copy_without_metadata(self):
"""Test copying checkouts from git."""
os.mkdir(self.codir)
subprocess.run(['git', 'init', '-q'], cwd=self.codir, check=True)
self.co_file_write('gitfile', 'gitfile contents')
subprocess.run(['git', 'add', '.'], cwd=self.codir, check=True)
subprocess.run(['git', 'commit', '-q', '-m', 'commit message'],
cwd=self.codir,
check=True)
vc_obj = GitVC(self.context, self.codir)
vc_obj.vc_checkout(self.srcdir, False)
os.chmod(os.path.join(self.srcdir, 'gitfile'), stat.S_IRUSR)
vc_obj.copy_without_metadata(self.srcdir, self.srcdir_copy)
self.assertEqual(read_files(self.srcdir_copy),
(set(), {
'gitfile': 'gitfile contents'
}, {}))
mode = stat.S_IMODE(
os.stat(os.path.join(self.srcdir_copy, 'gitfile')).st_mode)
self.assertEqual(mode, (stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP
| stat.S_IROTH))
def test_hard_link_files(self):
"""Test the hard_link_files function."""
create_files(self.indir, ['a', 'b', 'b/c'],
{'a1': 'a', 'a2': 'a', 'b/c/a3': 'a', 'b/a4': 'a',
'b1': 'b', 'b/b2': 'b', 'c': 'c'},
{'a-link': 'a1', 'dead-link': 'bad'})
os.chmod(os.path.join(self.indir, 'a1'), stat.S_IRWXU)
os.chmod(os.path.join(self.indir, 'b/c/a3'), stat.S_IRWXU)
os.chmod(os.path.join(self.indir, 'a2'), stat.S_IRUSR)
os.chmod(os.path.join(self.indir, 'b/a4'), stat.S_IRUSR)
hard_link_files(self.context, self.indir)
self.assertEqual(read_files(self.indir),
({'a', 'b', 'b/c'},
{'a1': 'a', 'a2': 'a', 'b/c/a3': 'a', 'b/a4': 'a',
'b1': 'b', 'b/b2': 'b', 'c': 'c'},
{'a-link': 'a1', 'dead-link': 'bad'}))
stat_a1 = os.stat(os.path.join(self.indir, 'a1'))
self.assertEqual(stat.S_IMODE(stat_a1.st_mode), stat.S_IRWXU)
stat_a3 = os.stat(os.path.join(self.indir, 'b/c/a3'))
self.assertEqual(stat.S_IMODE(stat_a3.st_mode), stat.S_IRWXU)
stat_a2 = os.stat(os.path.join(self.indir, 'a2'))
self.assertEqual(stat.S_IMODE(stat_a2.st_mode), stat.S_IRUSR)
stat_a4 = os.stat(os.path.join(self.indir, 'b/a4'))
self.assertEqual(stat.S_IMODE(stat_a4.st_mode), stat.S_IRUSR)
self.assertEqual(stat_a1.st_nlink, 2)
self.assertEqual(stat_a2.st_nlink, 2)
self.assertEqual(stat_a3.st_nlink, 2)
self.assertEqual(stat_a4.st_nlink, 2)
self.assertEqual(stat_a1.st_dev, stat_a3.st_dev)
self.assertEqual(stat_a1.st_ino, stat_a3.st_ino)
self.assertEqual(stat_a2.st_dev, stat_a4.st_dev)
self.assertEqual(stat_a2.st_ino, stat_a4.st_ino)
stat_b1 = os.stat(os.path.join(self.indir, 'b1'))
stat_b2 = os.stat(os.path.join(self.indir, 'b/b2'))
self.assertEqual(stat_b1.st_nlink, 2)
self.assertEqual(stat_b2.st_nlink, 2)
self.assertEqual(stat_b1.st_dev, stat_b2.st_dev)
self.assertEqual(stat_b1.st_ino, stat_b2.st_ino)
def test_run_build(self):
"""Test run_build, simple successful build."""
self.setup_rc('cfg.add_component("build_test")\n'
'cfg.multilibs.set((Multilib("build_test", '
'"build_test", ()), Multilib("build_test", '
'"build_test", ("-mtest",), osdir="test")))\n')
with self.redirect_stdout_stderr():
self.build_context.run_build()
stdout, stderr = self.stdout_stderr_read()
hosts = self.relcfg.hosts.get()
host_b0 = hosts[0].build_cfg
host_b1 = hosts[1].build_cfg
h0_all_objdir = self.relcfg.objdir_path(host_b0, 'build_test-all')
h0_all_objdir2 = self.relcfg.objdir_path(host_b0, 'build_test-all2')
h1_all_objdir = self.relcfg.objdir_path(host_b1, 'build_test-all')
h1_all_objdir2 = self.relcfg.objdir_path(host_b1, 'build_test-all2')
self.assertEqual(read_files(h0_all_objdir), (set(), {
'out1': 'all-hosts\n',
'out2': 'all-hosts-2\n'
}, {}))
self.assertEqual(read_files(h0_all_objdir2), (set(), {}, {}))
self.assertEqual(read_files(h1_all_objdir), (set(), {
'out1': 'all-hosts\n',
'out2': 'all-hosts-2\n'
}, {}))
self.assertEqual(read_files(h1_all_objdir2), (set(), {}, {}))
first_objdir = self.relcfg.objdir_path(host_b0, 'build_test-first')
first_objdir2 = self.relcfg.objdir_path(host_b0, 'build_test-first2')
self.assertEqual(read_files(first_objdir), ({'x', 'x/y'}, {
'GNUmakefile': 'all:; echo first-host $(X) > out\n',
'out': 'first-host Y\n'
}, {}))
self.assertEqual(read_files(first_objdir2), ({'x', 'x/y'}, {}, {}))
other_objdir = self.relcfg.objdir_path(host_b1, 'build_test-other')
self.assertEqual(read_files(other_objdir), (set(), {
'out': 'test python\n'
}, {}))
first_objdir_m0 = self.relcfg.objdir_path(
host_b0, 'build_test-first-aarch64-linux-gnu')
first_objdir_m1 = self.relcfg.objdir_path(
host_b0, 'build_test-first-aarch64-linux-gnu-mtest')
other_objdir_m0 = self.relcfg.objdir_path(
host_b1, 'build_test-other-aarch64-linux-gnu')
other_objdir_m1 = self.relcfg.objdir_path(
host_b1, 'build_test-other-aarch64-linux-gnu-mtest')
self.assertEqual(read_files(first_objdir_m0),
(set(), {
'out': 'aarch64-linux-gnu\n'
}, {}))
self.assertEqual(read_files(first_objdir_m1),
(set(), {
'out': 'aarch64-linux-gnu-mtest\n'
}, {}))
self.assertEqual(read_files(other_objdir_m0),
(set(), {
'out': 'test aarch64-linux-gnu\n'
}, {}))
self.assertEqual(read_files(other_objdir_m1),
(set(), {
'out': 'test aarch64-linux-gnu-mtest\n'
}, {}))
init_objdir = self.relcfg.objdir_path(None, 'build_test-init')
host_indep_objdir = self.relcfg.objdir_path(None,
'build_test-host-indep')
fini_objdir = self.relcfg.objdir_path(None, 'build_test-fini')
self.assertEqual(read_files(init_objdir), (set(), {
'out': 'init\n'
}, {}))
self.assertEqual(read_files(host_indep_objdir), (set(), {
'out': 'host-indep\n'
}, {}))
self.assertEqual(read_files(fini_objdir), (set(), {
'out': 'fini\n'
}, {}))
self.assertEqual(stdout, '')
lines = stderr.splitlines()
self.assertEqual(len(lines), 36)
for line in lines:
self.assertRegex(
line, r'^\[[0-2][0-9]:[0-5][0-9]:[0-6][0-9]\] '
r'\[00[0-1][0-9]/0018\] /.*(start|end)\Z')
self.assertIn('[0001/0018]', stderr)
self.assertIn('[0018/0018]', stderr)
self.assertIn('/x86_64-linux-gnu/all-hosts start', stderr)
self.assertIn('/x86_64-w64-mingw32/other-hosts end', stderr)
self.assertIn('/install-trees-x86_64-linux-gnu/package-input start',
stderr)
self.assertIn('/install-trees-x86_64-w64-mingw32/package-input end',
stderr)
self.assertIn('/x86_64-linux-gnu/package-output start', stderr)
self.assertIn('/x86_64-w64-mingw32/package-output end', stderr)
self.assertIn('/x86_64-linux-gnu/package-tar-xz end', stderr)
self.assertIn('/x86_64-w64-mingw32/package-tar-xz start', stderr)
self.assertIn('/init/init start', stderr)
self.assertIn('/init/pkgdir end', stderr)
self.assertIn('/host-indep/host-indep start', stderr)
self.assertIn('/fini/fini end', stderr)
# In this case, the created packages are empty.
pkg_0 = self.relcfg.pkgdir_path(hosts[0], '.tar.xz')
pkg_1 = self.relcfg.pkgdir_path(hosts[1], '.tar.xz')
dir_out = os.path.join(self.tempdir, 'toolchain-1.0')
subprocess.run(['tar', '-x', '-f', pkg_0],
cwd=self.tempdir,
check=True)
self.assertEqual(read_files(dir_out), (set(), {}, {}))
shutil.rmtree(dir_out)
subprocess.run(['tar', '-x', '-f', pkg_1],
cwd=self.tempdir,
check=True)
self.assertEqual(read_files(dir_out), (set(), {}, {}))
