def test_node_filter_dirname(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
build_dir = os.path.join(tmp_dir, 'output')
num_sources = 3
sources = self.generate_source_files(tmp_dir, num_sources, 200)
# set_event_settings( EventSettings( brief = False,
# with_output = True ) )
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
cfg.debug_backtrace = True
prj = Project(cfg)
node = prj.tools.CopyFiles(sources[0], target=tmp_install_dir)
prj.tools.CopyFiles(sources[1:], target=prj.node_dirname(node))
self.build_prj(prj, 3)
for src in sources:
tgt = os.path.join(tmp_install_dir, os.path.basename(src))
self.assertTrue(os.path.isfile(tgt))
node = prj.tools.CopyFiles(sources[0], target=tmp_install_dir)
prj.tools.CopyFiles(sources[1:], target=prj.node_dirname(node))
self.build_prj(prj, 0)
def test_temp_dir(self):
with Tempdir() as tmp_dir:
tmp_dir = Tempdir(root_dir=tmp_dir)
for i in range(10):
Tempfile(root_dir=tmp_dir, suffix='.tmp').close()
self.assertFalse(os.path.exists(tmp_dir))
def test_zip_files(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
# tmp_install_dir = Tempdir()
# tmp_dir = Tempdir()
sub_dir1 = Tempdir(root_dir=tmp_dir)
sub_dir2 = Tempdir(root_dir=tmp_dir)
build_dir = os.path.join(tmp_dir, 'output')
num_sources = 3
sources = []
sources += self.generate_source_files(sub_dir1, num_sources, 2)
sources += self.generate_source_files(sub_dir2, num_sources, 2)
zip_file = tmp_install_dir + "/test.zip"
cfg = ProjectConfig(args=["--bt", "build_dir=%s" % build_dir])
prj = Project(cfg)
value = prj.make_entity(name="test_content.txt",
data="To add to a ZIP file")
rename = [('test_file', sources[0])]
prj.tools.CreateZip(sources,
value,
target=zip_file,
basedir=tmp_dir,
rename=rename)
self.build_prj(prj, 1)
prj.tools.CreateZip(sources,
value,
target=zip_file,
basedir=tmp_dir,
rename=rename)
self.build_prj(prj, 0)
self.regenerate_file(sources[-1], 200)
prj.tools.CreateZip(sources,
value,
target=zip_file,
basedir=tmp_dir,
rename=rename)
self.build_prj(prj, 1)
def test_copy_files(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
# tmp_install_dir = Tempdir()
# tmp_dir = Tempdir()
sub_dir1 = Tempdir(root_dir=tmp_dir)
sub_dir2 = Tempdir(root_dir=tmp_dir)
num_sources = 3
sources = []
sources += self.generate_source_files(sub_dir1, num_sources, 2)
sources += self.generate_source_files(sub_dir2, num_sources, 2)
sources += self.generate_source_files(tmp_dir, num_sources, 2)
build_dir = os.path.join(tmp_dir, 'output')
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
prj = Project(cfg)
node = prj.tools.CopyFiles(sources,
target=tmp_install_dir,
basedir=tmp_dir)
node.options.batch_groups = 1
self.build_prj(prj, 1)
found_dirs = set()
files = find_files(tmp_install_dir, found_dirs=found_dirs)
filenames = set(os.path.basename(path) for path in files)
srcnames = set(os.path.basename(path) for path in sources)
self.assertSetEqual(filenames, srcnames)
found_dir_names = set(
os.path.basename(path) for path in found_dirs)
sub_dir_names = set(
os.path.basename(path) for path in [sub_dir1, sub_dir2])
self.assertSetEqual(found_dir_names, sub_dir_names)
prj.tools.CopyFiles(sources,
target=tmp_install_dir,
basedir=tmp_dir)
self.build_prj(prj, 0)
def test_bm_no_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 256)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
node1 = Node(builder1, node1)
node2 = Node(builder2, node2)
bm.add([node1, node2])
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files * 2)
def test_bm_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
src_files = self.generate_source_files(tmp_dir, 3, 201)
builder = ChecksumBuilder(options, 0, 256, replace_ext=True)
self.building_nodes = self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.building_nodes, 2)
self.assertEqual(self.building_nodes, self.built_nodes)
self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.built_nodes, 0)
bm = _add_nodes_to_bm(builder, src_files)
try:
bm.clear()
bm.self_test()
finally:
bm.close()
def test_bm_node_index(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 2
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
node = Node(builder, src_files)
nodes = [
Node(builder, node[i * 2]) for i in range(num_src_files + 1)
]
node2 = Node(builder, node[1:2][:])
#
bm.add([node2])
bm.add(nodes)
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files + 1 + 1)
def test_prj_list_options(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir, "test", "run"])
prj = Project(cfg)
self.assertTrue(prj.list_options())
self.assertTrue(prj.list_options(brief=True))
self.assertFalse(prj.list_tools_options('c++'))
def test_prj_implicit_value_args(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir])
prj = Project(cfg)
tool = prj.tools.add_tool(_NullTool)
tool.noop(v1="a", v2="b", v3="c")
prj.build()
self.assertEqual(self.built_nodes, 1)
# -----------------------------------------------------------
self.built_nodes = 0
tool.noop(v1="aa", v2="bb", v3="cc")
prj.build()
self.assertEqual(self.built_nodes, 0)
# -----------------------------------------------------------
self.built_nodes = 0
v1 = SimpleEntity("a", name="value1")
tool.noop(v1=v1, v2="b", v3="c")
prj.build()
self.assertEqual(self.built_nodes, 1)
# -----------------------------------------------------------
self.built_nodes = 0
v1 = SimpleEntity("ab", name="value1")
tool.noop(v1=v1, v2="b", v3="c")
prj.build()
self.assertEqual(self.built_nodes, 1)
# -----------------------------------------------------------
self.built_nodes = 0
v1 = SimpleEntity("ab", name="value1")
tool.noop(v1=v1, v2="b", v3="c")
prj.build()
self.assertEqual(self.built_nodes, 0)
def test_copy_file_as(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
# tmp_install_dir = Tempdir()
# tmp_dir = Tempdir()
build_dir = os.path.join(tmp_dir, 'output')
source = self.generate_file(tmp_dir, 200)
target = os.path.join(tmp_install_dir, 'copy_as_source.dat')
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
prj = Project(cfg)
prj.tools.CopyFileAs(source, target=target)
self.build_prj(prj, 1)
prj.tools.CopyFileAs(source, target=target)
self.build_prj(prj, 0)
def test_write_file(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
# tmp_install_dir = Tempdir()
# tmp_dir = Tempdir()
build_dir = os.path.join(tmp_dir, 'output')
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
prj = Project(cfg)
buf = "Test buffer content"
target = os.path.join(tmp_install_dir, 'write_content.txt')
prj.tools.WriteFile(buf, target=target)
self.build_prj(prj, 1)
prj.tools.WriteFile(buf, target=target)
self.build_prj(prj, 0)
def test_bm_skip_nodes_by_value(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
node = Node(ValueBuilder(options), [1, 2, 3, 4])
bm.add([node])
bm.build_if(False, node)
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 0)
def test_bm_dup_names(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
# options.batch_groups = 4
options.batch_size = 3
src_files = self.generate_source_files(tmp_dir, 10, 201)
builder = ChecksumBadBuilder(options, 0, 256, replace_ext=True)
self.building_nodes = self.built_nodes = 0
with self.assertRaises(ErrorNodeDuplicateNames):
_build_checksums(builder, src_files, jobs=4)
def test_prj_expensive(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir])
prj = Project(cfg)
cmd_heavy = prj.tools.ExecuteCommand(
sys.executable, "-c", "print('test expensive')")
prj.tools.ExecuteCommand(sys.executable, "-c",
"print('test light')")
prj.expensive(cmd_heavy)
prj.build()
self.assertEqual(self.built_nodes, 2)
def test_bm_rebuild(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 10
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
def _build_nodes(num_dups, uptodate):
bm = BuildManager()
self.building_nodes = self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
src_entities = tuple(map(FileChecksumEntity, src_files))
num_built_nodes = 0
for i in range(num_dups):
num_built_nodes = 1
node = Node(builder, src_entities)
node = Node(builder, node)
num_built_nodes += 2
node = Node(builder, node)
num_built_nodes += 2**2
node = Node(builder, node)
num_built_nodes += 2**3
bm.add([node])
_build(bm, jobs=10, explain=False)
if uptodate:
num_built_nodes = 0
else:
num_built_nodes *= num_src_files
self.assertEqual(self.building_nodes, num_built_nodes)
_build_nodes(3, False)
_build_nodes(3, True)
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumBuilder(options, 0, 256)
options.batch_build = False
single_builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files + 1)
# -----------------------------------------------------------
self.regenerate_file(src_files[0], 201)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_find_files(self):
with Tempdir() as tmp_dir:
# tmp_dir = Tempdir()
# print("tmp_dir: %s" % tmp_dir)
build_dir = os.path.join(tmp_dir, 'output')
num_sources = 3
sources = self.generate_source_files(tmp_dir, num_sources, 20)
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
prj = Project(cfg)
prj.tools.FindFiles(tmp_dir)
prj.tools.FindFiles(tmp_dir, mask="*.tmp")
self.build_prj(prj, 2)
prj.tools.FindFiles(tmp_dir)
prj.tools.FindFiles(tmp_dir, mask="*.tmp")
self.build_prj(prj, 0)
prj.tools.FindFiles(tmp_dir, exclude_mask=".*db*")
self.build_prj(prj, 1)
prj.tools.FindFiles(tmp_dir, exclude_mask=".*db*")
self.build_prj(prj, 0)
sources += self.generate_source_files(tmp_dir, 1, 20)
prj.tools.FindFiles(tmp_dir, exclude_mask=".*db*")
prj.tools.FindFiles(tmp_dir, mask="*.tmp")
self.build_prj(prj, 2)
prj.tools.FindFiles(tmp_dir, exclude_mask=".db*")
prj.tools.FindFiles(tmp_dir, mask="*.tmp")
self.clear_prj(prj)
self.assertTrue(all(os.path.isfile(source) for source in sources))
def test_prj_default_targets(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir])
prj = Project(cfg)
prj.tools.ExecuteCommand(
sys.executable, "-c", "print('test builtin')")
cmd_other = prj.tools.ExecuteCommand(
sys.executable, "-c", "print('test other')")
cmd_other2 = prj.tools.ExecuteCommand(
sys.executable, "-c", "print('test other2')")
prj.default_build([cmd_other, cmd_other2])
prj.build()
self.assertEqual(self.built_nodes, 2)
def test_node_batch(self):
with Tempdir() as tmp_dir:
vfile_name = Tempfile(root_dir=tmp_dir)
vfile_name.close()
with EntitiesFile(vfile_name) as vfile:
src_files = self.generate_source_files(tmp_dir, 5, 100)
self._rebuild_batch_node(vfile, src_files, len(src_files))
self._rebuild_batch_node(vfile, src_files, 0)
self._rebuild_batch_node(vfile, src_files[:-2], 0)
self._rebuild_batch_node(vfile, src_files[0:1], 0)
# -----------------------------------------------------------
write_bin_file(src_files[1], b"src_file1")
write_bin_file(src_files[2], b"src_file1")
self._rebuild_batch_node(vfile, src_files, 2)
def test_bm_expensive(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
event = threading.Event()
heavy = ExpensiveValueBuilder(options, event, do_expensive=True)
light = ExpensiveValueBuilder(options, event, do_expensive=False)
node1 = Node(heavy, [1, 2, 3, 4, 5, 7])
node2 = Node(light, list(range(10, 100, 10)))
bm.add([node2, node1])
bm.expensive(node1)
_build(bm, jobs=16)
def test_bm_node_build_fail(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
builder = FailedBuilder(options)
nodes = [
Node(builder, SimpleEntity("123-%s" % (i, ))) for i in range(4)
]
bm.add(nodes)
self.assertRaises(Exception, _build, bm)
self.assertEqual(self.built_nodes, 0)
def test_prj_targets(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir, "test", "run"])
prj = Project(cfg)
cmd = prj.tools.ExecuteCommand(
sys.executable, "-c", "print('test builtin')")
prj.tools.ExecuteCommand(sys.executable, "-c",
"print('test other')")
self.assertSequenceEqual(prj.get_build_targets(), ['test', 'run'])
prj.alias_nodes(prj.get_build_targets(), cmd)
prj.build()
self.assertEqual(self.built_nodes, 1)
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [
Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])
]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def test_exec(self):
with Tempdir() as tmp_dir:
build_dir = os.path.join(tmp_dir, 'build')
options = builtin_options()
cmd = [sys.executable, '-c', 'print("TEST EXEC")']
options.build_dir = build_dir
exec_cmd = ExecuteCommandBuilder(options)
bm = BuildManager()
try:
result = Node(exec_cmd, cmd)
bm.add([result])
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 1)
self.assertEqual(self.building_started, self.built_nodes)
bm.close()
result = Node(exec_cmd, cmd)
bm = BuildManager()
bm.add([result])
self.building_started = 0
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 0)
finally:
bm.close()
def test_bm_build(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
src_files = self.generate_source_files(tmp_dir, 5, 201)
builder = ChecksumBuilder(options, 0, 256)
self.building_nodes = self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.building_nodes, 2)
self.assertEqual(self.building_nodes, self.built_nodes)
# -----------------------------------------------------------
self.building_nodes = self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.building_nodes, 0)
self.assertEqual(self.building_nodes, self.built_nodes)
# -----------------------------------------------------------
builder = ChecksumBuilder(options, 32, 1024)
self.building_nodes = self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.building_nodes, 2)
self.assertEqual(self.building_nodes, self.built_nodes)
# -----------------------------------------------------------
self.building_nodes = self.built_nodes = 0
_build_checksums(builder, src_files)
self.assertEqual(self.building_nodes, 0)
self.assertEqual(self.building_nodes, self.building_nodes)
def test_bm_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 1024)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
# node1 = Node( builder1, node1 )
# node2 = Node( builder2, node2 )
bm.add([node1, node2])
self.assertRaises(ErrorNodeSignatureDifferent, _build, bm)
def test_bm_skip_nodes_by_node(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node1 = Node(CondBuilder(options), False)
node1 = Node(ValueBuilder(options), [1, 2])
cond_node2 = Node(CondBuilder(options), True)
node2 = Node(ValueBuilder(options), [3, 4])
main = Node(ValueBuilder(options), [7, 8, node1, node2])
bm.add([main])
bm.build_if(cond_node1, node1)
bm.build_if(cond_node2, node2)
_build(bm, jobs=4)
self.assertEqual(main.get(), "7-8-3-4")
self.assertEqual(self.built_nodes, 4)
def test_prj_builtin_tools(self):
with Tempdir() as tmp_dir:
cfg = ProjectConfig(args=["build_dir=%s" % tmp_dir])
prj = Project(cfg)
cmd = sys.executable, "-c", "print('test builtin')"
prj.tools.ExecuteCommand(cmd)
prj.build()
self.assertEqual(self.building_started, 1)
self.assertEqual(self.built_nodes, 1)
self.building_started = 0
prj = Project(cfg)
prj.tools.ExecuteCommand(cmd)
prj.build()
self.assertEqual(self.building_started, 0)
def test_exec_method(self):
def copy_file_ext(builder, source_entities, targets, ext):
src_file = source_entities[0].get()
dst_file = os.path.splitext(src_file)[0] + ext
shutil.copy(src_file, dst_file)
targets.add_targets(dst_file)
with Tempdir() as tmp_dir:
set_event_settings(
EventSettings(brief=True, with_output=True, trace_exec=False))
build_dir = os.path.join(tmp_dir, 'build_output')
num_sources = 2
sources = self.generate_source_files(tmp_dir,
num_sources,
size=200,
suffix='.cpp')
headers = self.generate_source_files(tmp_dir,
num_sources,
size=200,
suffix='.hpp')
cfg = ProjectConfig(args=["build_dir=%s" % build_dir])
prj = Project(cfg)
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ))
prj.tools.ExecuteMethod(headers,
method=copy_file_ext,
args=('.hxx', ))
self.build_prj(prj, len(sources) + len(headers))
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ))
prj.tools.ExecuteMethod(headers,
method=copy_file_ext,
args=('.hxx', ))
self.build_prj(prj, 0)
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cc', ))
self.build_prj(prj, len(sources))
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ))
self.build_prj(prj, len(sources))
# -----------------------------------------------------------
for src in sources:
self.assertTrue(
os.path.isfile(os.path.splitext(src)[0] + '.cxx'))
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ))
self.clear_prj(prj)
for src in sources:
self.assertFalse(
os.path.isfile(os.path.splitext(src)[0] + '.cxx'))
# -----------------------------------------------------------
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ))
self.build_prj(prj, len(sources))
for src in sources:
self.assertTrue(
os.path.isfile(os.path.splitext(src)[0] + '.cxx'))
prj.tools.ExecuteMethod(sources,
method=copy_file_ext,
args=('.cxx', ),
clear_targets=False)
self.clear_prj(prj)
for src in sources:
self.assertTrue(
os.path.isfile(os.path.splitext(src)[0] + '.cxx'))