def test_bm_node_index(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 2
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
node = Node(builder, src_files)
nodes = [
Node(builder, node[i * 2]) for i in range(num_src_files + 1)
]
node2 = Node(builder, node[1:2][:])
#
bm.add([node2])
bm.add(nodes)
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files + 1 + 1)
def _build_nodes(num_dups, uptodate):
bm = BuildManager()
self.building_nodes = self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
src_entities = tuple(map(FileChecksumEntity, src_files))
num_built_nodes = 0
for i in range(num_dups):
num_built_nodes = 1
node = Node(builder, src_entities)
node = Node(builder, node)
num_built_nodes += 2
node = Node(builder, node)
num_built_nodes += 2**2
node = Node(builder, node)
num_built_nodes += 2**3
bm.add([node])
_build(bm, jobs=10, explain=False)
if uptodate:
num_built_nodes = 0
else:
num_built_nodes *= num_src_files
self.assertEqual(self.building_nodes, num_built_nodes)
def test_node_speed( self ):
try:
tmp_files = []
source_files = _generateFiles( tmp_files, 4000, 50 * 1024 )
idep_files = _copyFiles( tmp_files, source_files, 'h' )
with Tempfile() as tmp:
vfile = ValuesFile( tmp )
try:
builder = TestSpeedBuilder("TestSpeedBuilder", "tmp", "h")
for source in source_files:
node = Node( builder, _FileValueType( source ) )
node.initiate()
self.assertFalse( node.checkActual( vfile ) )
builder.build( node )
builder.save( vfile, node )
for tmp_file in node.target_values:
tmp_files.append( tmp_file.name )
t = lambda vfile = vfile, builder = builder, source_files = source_files, testNoBuildSpeed = _testNoBuildSpeed: testNoBuildSpeed( vfile, builder, source_files )
t = timeit.timeit( t, number = 1 )
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove( tmp_file )
except OSError:
pass
def test_bm_no_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 256)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
node1 = Node(builder1, node1)
node2 = Node(builder2, node2)
bm.add([node1, node2])
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files * 2)
def _rebuild_batch_node(self, vfile, src_files, built_count):
options = builtin_options()
options.batch_build = True
options.batch_groups = 2
builder = CopyBuilder(options, "tmp", "i")
node = Node(builder, src_files)
dep = SimpleEntity("11", name="dep1")
node.depends(dep)
node.initiate()
split_nodes = node.build_split(vfile, False)
if built_count == 0:
self.assertFalse(split_nodes)
self.assertTrue(node.check_actual(vfile, False))
else:
for split_node in split_nodes:
self.assertFalse(split_node.check_actual(vfile, False))
split_node.build()
split_node.save(vfile)
self.assertEqual(len(split_nodes), 2)
self.assertTrue(node.check_actual(vfile, False))
def _makeNodes( builder ): node1 = Node( builder, value1 ) copy_node1 = Node( builder, node1 ) copy2_node1 = Node( builder, copy_node1 ) node2 = Node( builder, value2 ) node3 = Node( builder, value3 ) copy_node3 = Node( builder, node3 ) copy2_node3 = Node( builder, copy_node3 ) copy2_node3.depends( [node1, copy_node1] ) return node1, node2, node3, copy_node1, copy_node3, copy2_node1, copy2_node3
def _make_nodes(builder):
node1 = Node(builder, value1)
copy_node1 = Node(builder, node1)
copy2_node1 = Node(builder, copy_node1)
node2 = Node(builder, value2)
node3 = Node(builder, value3)
copy_node3 = Node(builder, node3)
copy2_node3 = Node(builder, copy_node3)
copy2_node3.depends([node1, copy_node1])
return node1, node2, node3, copy_node1,\
copy_node3, copy2_node1, copy2_node3
def _rebuild_batch_node(self, vfile, src_files, built_count):
options = builtin_options()
options.batch_build = True
options.batch_groups = 2
builder = CopyBuilder(options, "tmp", "i")
node = Node(builder, src_files)
dep = SimpleEntity("11", name="dep1")
node.depends(dep)
node.initiate()
split_nodes = node.build_split(vfile, False)
if built_count == 0:
self.assertFalse(split_nodes)
self.assertTrue(node.check_actual(vfile, False))
else:
for split_node in split_nodes:
self.assertFalse(split_node.check_actual(vfile, False))
split_node.build()
split_node.save(vfile)
self.assertEqual(len(split_nodes), 2)
self.assertTrue(node.check_actual(vfile, False))
def _add_nodes_to_bm(builder, src_files):
bm = BuildManager()
try:
checksums_node = Node(builder, src_files)
checksums_node2 = Node(builder, checksums_node)
bm.add([checksums_node])
bm.self_test()
bm.add([checksums_node2])
bm.self_test()
except Exception:
bm.close()
raise
return bm
def __call__(self, *args, **kw):
options, deps, sources, builder_args = self._get_builder_args(kw)
sources += args
sources = flatten_list(sources)
builder = self.method(options, **builder_args)
node = Node(builder, sources)
node.depends(deps)
self.project.add_nodes((node,))
return node
def __call__(self, *args, **kw):
options, deps, sources, builder_args = self._get_builder_args(kw)
sources += args
sources = flatten_list(sources)
builder = self.method(options, **builder_args)
node = Node(builder, sources)
node.depends(deps)
self.project.add_nodes((node, ))
return node
def _add_sources():
if current_builder is None:
new_sources.extend(current_sources)
return
src_node = Node(current_builder, current_sources, cwd)
new_sources.append(src_node)
def _test_no_build_speed(vfile, builder, source_values):
for source in source_values:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
if not node.check_actual(vfile, False):
raise AssertionError("node is not actual")
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumBuilder(options, 0, 256)
options.batch_build = False
single_builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files + 1)
# -----------------------------------------------------------
self.touch_cpp_file(src_files[0])
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_bm_expensive(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
event = threading.Event()
heavy = ExpensiveValueBuilder(options, event, do_expensive=True)
light = ExpensiveValueBuilder(options, event, do_expensive=False)
node1 = Node(heavy, [1, 2, 3, 4, 5, 7])
node2 = Node(light, list(range(10, 100, 10)))
bm.add([node2, node1])
bm.expensive(node1)
_build(bm, jobs=16)
def test_bm_tags(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files * 2)
# -----------------------------------------------------------
self.regenerate_file(src_files[0], )
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_bm_tags(self):
with Tempdir() as tmp_dir:
options = builtinOptions()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generateSourceFiles( tmp_dir, num_src_files, 201 )
builder = ChecksumSingleBuilder( options, 0, 256 )
bm = BuildManager()
self.finished_nodes = 0
node = Node( builder, src_files )
node_md5 = Node( builder, node.at('md5') )
bm.add( node_md5 )
_build( bm )
self.assertEqual( self.finished_nodes, num_src_files * 2 )
#//-------------------------------------------------------//
self.touchCppFile( src_files[0] )
bm = BuildManager()
self.finished_nodes = 0
node = Node( builder, src_files )
node_md5 = Node( builder, node.at('md5') )
bm.add( node_md5 )
_build( bm )
self.assertEqual( self.finished_nodes, 2 )
def test_bm_deps_speed(self):
bm = BuildManager()
value = SimpleEntity("http://aql.org/download", name="target_url1")
builder = CopyValueBuilder()
node = Node(builder, value)
bm.add([node])
_generate_node_tree(bm, builder, node, 5000)
def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def _test_no_build_speed(vfile, builder, source_values):
for source in source_values:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
if not node.check_actual(vfile, False):
raise AssertionError("node is not actual")
def test_exec(self):
with Tempdir() as tmp_dir:
build_dir = os.path.join(tmp_dir, 'build')
options = builtin_options()
cmd = [sys.executable, '-c', 'print("TEST EXEC")']
options.build_dir = build_dir
exec_cmd = ExecuteCommandBuilder(options)
bm = BuildManager()
try:
result = Node(exec_cmd, cmd)
bm.add([result])
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 1)
self.assertEqual(self.building_started, self.built_nodes)
bm.close()
result = Node(exec_cmd, cmd)
bm = BuildManager()
bm.add([result])
self.building_started = 0
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 0)
finally:
bm.close()
def test_bm_skip_nodes_by_node(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node1 = Node(CondBuilder(options), False)
node1 = Node(ValueBuilder(options), [1, 2])
cond_node2 = Node(CondBuilder(options), True)
node2 = Node(ValueBuilder(options), [3, 4])
main = Node(ValueBuilder(options), [7, 8, node1, node2])
bm.add([main])
bm.build_if(cond_node1, node1)
bm.build_if(cond_node2, node2)
_build(bm, jobs=4)
self.assertEqual(main.get(), "7-8-3-4")
self.assertEqual(self.built_nodes, 4)
def test_bm_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 1024)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
# node1 = Node( builder1, node1 )
# node2 = Node( builder2, node2 )
bm.add([node1, node2])
self.assertRaises(ErrorNodeSignatureDifferent, _build, bm)
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumBuilder(options, 0, 256)
options.batch_build = False
single_builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files + 1)
# -----------------------------------------------------------
self.regenerate_file(src_files[0], 201)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_bm_skip_nodes_by_value(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
node = Node(ValueBuilder(options), [1, 2, 3, 4])
bm.add([node])
bm.build_if(False, node)
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 0)
def test_bm_node_build_fail(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
builder = FailedBuilder(options)
nodes = [
Node(builder, SimpleEntity("123-%s" % (i, ))) for i in range(4)
]
bm.add(nodes)
self.assertRaises(Exception, _build, bm)
self.assertEqual(self.built_nodes, 0)
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [
Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])
]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
def test_bm_skip_nodes_by_node(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node1 = Node(CondBuilder(options), False)
node1 = Node(ValueBuilder(options), [1, 2])
cond_node2 = Node(CondBuilder(options), True)
node2 = Node(ValueBuilder(options), [3, 4])
main = Node(ValueBuilder(options), [7, 8, node1, node2])
bm.add([main])
bm.build_if(cond_node1, node1)
bm.build_if(cond_node2, node2)
_build(bm, jobs=4)
self.assertEqual(main.get(), "7-8-3-4")
self.assertEqual(self.built_nodes, 4)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic,
_cyclic_deps, node4, node3)
def test_bm_sync_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(
SyncValueBuilder(options,
name="31",
number=0,
sleep_interval=0), SimpleEntity("31"))
node32 = Node(
SyncValueBuilder(options,
name="32",
number=0,
sleep_interval=0), SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31, ))
node22 = Node(
SyncValueBuilder(options,
name="22",
number=0,
sleep_interval=5), (node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33, ))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33, ))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
bm.add((node10, node11))
bm.sync((node10, node11), deep=True)
_build(bm, jobs=4)
def _rebuild_node(self, vfile, builder, values, deps, tmp_files):
node = Node(builder, values)
node.depends(deps)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, values)
node.depends(deps)
node.initiate()
node.build_split(vfile, explain=True)
self.assertTrue(node.check_actual(vfile, explain=True))
tmp_files.extend(target.get() for target in node.get_target_entities())
tmp_files.extend(target.get()
for target in node.get_side_effect_entities())
return node
def _generate_node_tree(bm, builder, node, depth):
while depth:
node = Node(builder, node)
bm.add([node])
depth -= 1
def test_node_file(self):
try:
tmp_files = []
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
with Tempfile(suffix='.1') as tmp1:
with Tempfile(suffix='.2') as tmp2:
value1 = FileChecksumEntity(tmp1)
value2 = FileChecksumEntity(tmp2)
options = builtin_options()
builder = CopyBuilder(options, "tmp", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "d")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
tmp1.write(b'123')
tmp1.flush()
value1 = FileChecksumEntity(tmp1)
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
with Tempfile(suffix='.3') as tmp3:
value3 = FileChecksumEntity(tmp3)
node3 = self._rebuild_node(
vfile, builder, [value3], [], tmp_files)
node = self._rebuild_node(vfile, builder,
[value1, node3], [],
tmp_files)
builder3 = CopyBuilder(options, "xxx", "3")
node3 = self._rebuild_node(vfile, builder3,
[value3], [],
tmp_files)
# node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx,
# ,tmp3.3
node = self._rebuild_node(vfile, builder,
[value1, node3], [],
tmp_files)
# node: CopyBuilder, tmp, i, tmp1, tmp3.xxx ->
# tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
dep = SimpleEntity("1", name="dep1")
node = self._rebuild_node(vfile, builder,
[value1, node3],
[dep], tmp_files)
dep = SimpleEntity("11", name="dep1")
node = self._rebuild_node(vfile, builder,
[value1, node3],
[dep], tmp_files)
node3 = self._rebuild_node(vfile, builder3,
[value1], [],
tmp_files)
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
node3 = self._rebuild_node(vfile, builder3,
[value2], [],
tmp_files)
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
node_tname = node.get_target_entities()[0].name
with open(node_tname, 'wb') as f:
f.write(b'333')
f.flush()
node = self._rebuild_node(vfile, builder,
[value1], [node3],
tmp_files)
side_effects = node.get_side_effect_entities()
with open(side_effects[0].name, 'wb') as f:
f.write(b'abc')
f.flush()
node = Node(builder, [value1])
node.depends([node3])
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile,
False))
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_value(self):
with Tempfile() as tmp:
vfile = ValuesFile( tmp )
try:
value1 = SimpleValue( "http://aql.org/download1", name = "target_url1" )
value2 = SimpleValue( "http://aql.org/download2", name = "target_url2" )
value3 = SimpleValue( "http://aql.org/download3", name = "target_url3" )
options = builtinOptions()
builder = ChecksumBuilder( options )
node = Node( builder, [value1, value2, value3] )
node.initiate()
self.assertFalse( node.checkActual( vfile ) )
node.build()
node.save( vfile )
self.assertTrue( node.checkActual( vfile ) )
node = Node( builder, [value1, value2, value3] )
node.initiate()
self.assertTrue( node.checkActual( vfile ) )
node.build()
node.save( vfile )
self.assertTrue( node.checkActual( vfile ) )
node = Node( builder, [value1, value2, value3] )
node.depends( NullValue() )
node.initiate()
self.assertFalse( node.checkActual( vfile ) )
node.build()
node.save( vfile )
self.assertFalse( node.checkActual( vfile ) )
finally:
vfile.close()
def test_bm_require_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(
SyncValueBuilder(options,
name="31",
number=0,
sleep_interval=0), SimpleEntity("31"))
node32 = Node(
SyncValueBuilder(options,
name="32",
number=0,
sleep_interval=0), SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31, ))
node22 = Node(
SyncValueBuilder(options,
name="22",
number=0,
sleep_interval=5), (node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33, ))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33, ))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
# print( "node30: %s" % node30 )
# print( "node31: %s" % node31 )
# print( "node32: %s" % node32 )
# print( "node33: %s" % node33 )
#
# print( "node20: %s" % node20 )
# print( "node21: %s" % node21 )
# print( "node22: %s" % node22 )
# print( "node23: %s" % node23 )
# print( "node24: %s" % node24 )
#
# print( "node10: %s" % node10 )
# print( "node11: %s" % node11 )
bm.add((node10, node11))
bm.module_depends(node10, [node11])
_build(bm, jobs=4)
def test_node_speed(self):
try:
tmp_files = []
source_files = _generate_files(tmp_files, 4000, 50 * 1024)
# idep_files = _copy_files(tmp_files, source_files, 'h')
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
builder = _SpeedBuilder("SpeedBuilder", "tmp", "h")
for source in source_files:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
for tmp_file in node.get_targets():
tmp_files.append(tmp_file)
t = lambda: _test_no_build_speed(vfile, builder,
source_files)
t = timeit.timeit(t, number=1)
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleValue( "http://aql.org/download1", name = "target_url1" )
value2 = SimpleValue( "http://aql.org/download2", name = "target_url2" )
value3 = SimpleValue( "http://aql.org/download3", name = "target_url3" )
options = builtinOptions()
builder = CopyValueBuilder( options )
node0 = Node( builder, value1 )
node1 = Node( builder, node0 )
node2 = Node( builder, node1 )
node3 = Node( builder, value2 )
node4 = Node( builder, value3 )
node5 = Node( builder, node4 )
node6 = Node( builder, node5 )
node6.depends( [node0, node1] )
bm.add( node0 ); bm.selfTest(); self.assertEqual( len(bm), 1 )
bm.add( node1 ); bm.selfTest(); self.assertEqual( len(bm), 2 )
bm.add( node2 ); bm.selfTest(); self.assertEqual( len(bm), 3 )
bm.add( node3 ); bm.selfTest(); self.assertEqual( len(bm), 4 )
bm.add( node4 ); bm.selfTest(); self.assertEqual( len(bm), 5 )
bm.add( node5 ); bm.selfTest(); self.assertEqual( len(bm), 6 )
bm.add( node6 ); bm.selfTest(); self.assertEqual( len(bm), 7 )
node0.depends( node3 ); bm.depends( node0, node3 ); bm.selfTest()
node1.depends( node3 ); bm.depends( node1, node3 ); bm.selfTest()
node2.depends( node3 ); bm.depends( node2, node3 ); bm.selfTest()
node3.depends( node4 ); bm.depends( node3, node4 ); bm.selfTest()
node0.depends( node5 ); bm.depends( node0, node5 ); bm.selfTest()
node5.depends( node3 ); bm.depends( node5, node3 ); bm.selfTest()
with self.assertRaises(ErrorNodeDependencyCyclic):
node4.depends( node3 ); bm.depends( node4, node3 ); bm.selfTest()
def test_node_file(self):
try:
tmp_files = []
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
with Tempfile(suffix='.1') as tmp1:
with Tempfile(suffix='.2') as tmp2:
value1 = FileChecksumEntity(tmp1)
value2 = FileChecksumEntity(tmp2)
options = builtin_options()
builder = CopyBuilder(options, "tmp", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "i")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
builder = CopyBuilder(options, "ttt", "d")
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
tmp1.write(b'123')
tmp1.flush()
value1 = FileChecksumEntity(tmp1)
node = self._rebuild_node(vfile, builder,
[value1, value2], [],
tmp_files)
with Tempfile(suffix='.3') as tmp3:
value3 = FileChecksumEntity(tmp3)
node3 = self._rebuild_node(
vfile, builder, [value3], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1, node3], [],
tmp_files)
builder3 = CopyBuilder(options, "xxx", "3")
node3 = self._rebuild_node(
vfile, builder3, [value3], [], tmp_files)
# node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx,
# ,tmp3.3
node = self._rebuild_node(
vfile, builder, [value1, node3], [],
tmp_files)
# node: CopyBuilder, tmp, i, tmp1, tmp3.xxx ->
# tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
dep = SimpleEntity("1", name="dep1")
node = self._rebuild_node(
vfile, builder, [value1, node3], [dep],
tmp_files)
dep = SimpleEntity("11", name="dep1")
node = self._rebuild_node(
vfile, builder, [value1, node3], [dep],
tmp_files)
node3 = self._rebuild_node(
vfile, builder3, [value1], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
node3 = self._rebuild_node(
vfile, builder3, [value2], [], tmp_files)
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
node_tname = node.get_target_entities()[0].name
with open(node_tname, 'wb') as f:
f.write(b'333')
f.flush()
node = self._rebuild_node(
vfile, builder, [value1], [node3],
tmp_files)
side_effects = node.get_side_effect_entities()
with open(side_effects[0].name, 'wb') as f:
f.write(b'abc')
f.flush()
node = Node(builder, [value1])
node.depends([node3])
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(
vfile, False))
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_value(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
value1 = SimpleEntity(
"http://aql.org/download1", name="target_url1")
value2 = SimpleEntity(
"http://aql.org/download2", name="target_url2")
value3 = SimpleEntity(
"http://aql.org/download3", name="target_url3")
dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value")
dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value")
options = builtin_options()
builder = ChecksumBuilder(options)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
finally:
vfile.close()
def _rebuildNode( self, vfile, builder, values, deps, tmp_files):
node = Node( builder, values )
node.depends( deps )
node.initiate()
self.assertFalse( node.checkActual( vfile ) )
node.build()
node.save( vfile )
self.assertTrue( node.checkActual( vfile ) )
node = Node( builder, values )
node.depends( deps )
node.initiate()
self.assertTrue( node.checkActual( vfile ) )
node.build()
node.save( vfile )
self.assertTrue( node.checkActual( vfile ) )
for tmp_file in node.getTargetValues():
tmp_files.append( tmp_file.name )
for tmp_file in node.getSideEffectValues():
tmp_files.append( tmp_file.name )
return node
def _rebuild_node(self, vfile, builder, values, deps, tmp_files):
node = Node(builder, values)
node.depends(deps)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, values)
node.depends(deps)
node.initiate()
node.build_split(vfile, explain=True)
self.assertTrue(node.check_actual(vfile, explain=True))
tmp_files.extend(target.get() for target in node.get_target_entities())
tmp_files.extend(target.get()
for target in node.get_side_effect_entities())
return node
def test_node_file(self):
try:
tmp_files = []
with Tempfile() as tmp:
vfile = ValuesFile( tmp )
try:
with Tempfile( suffix = '.1' ) as tmp1:
with Tempfile( suffix = '.2' ) as tmp2:
value1 = FileChecksumValue( tmp1 )
value2 = FileChecksumValue( tmp2 )
options = builtinOptions()
builder = CopyBuilder( options, "tmp", "i")
node = self._rebuildNode( vfile, builder, [value1, value2], [], tmp_files )
builder = CopyBuilder( options, "ttt", "i")
node = self._rebuildNode( vfile, builder, [value1, value2], [], tmp_files )
builder = CopyBuilder( options, "ttt", "d")
node = self._rebuildNode( vfile, builder, [value1, value2], [], tmp_files )
tmp1.write(b'123')
tmp1.flush()
value1 = FileChecksumValue( tmp1 )
node = self._rebuildNode( vfile, builder, [value1, value2], [], tmp_files )
with Tempfile( suffix = '.3' ) as tmp3:
value3 = FileChecksumValue( tmp3 )
node3 = self._rebuildNode( vfile, builder, [value3], [], tmp_files )
node = self._rebuildNode( vfile, builder, [value1, node3], [], tmp_files )
# node3: CopyBuilder, tmp, i, tmp3 -> tmp3.tmp, ,tmp3.i
# node: CopyBuilder, tmp, i, tmp1, tmp3.tmp -> tmp1.tmp, tmp3.tmp.tmp, , tmp1.i, tmp3.tmp.i
builder3 = CopyBuilder( options, "xxx", "3")
node3 = self._rebuildNode( vfile, builder3, [value3], [], tmp_files )
# node3: CopyBuilder, xxx, 3, tmp3 -> tmp3.xxx, ,tmp3.3
node = self._rebuildNode( vfile, builder, [value1, node3], [], tmp_files )
# node: CopyBuilder, tmp, i, tmp1, tmp3.xxx -> tmp1.tmp, tmp3.xxx.tmp, , tmp1.i, tmp3.xxx.i
node = self._rebuildNode( vfile, builder, [value1], [node3], tmp_files )
dep = SimpleValue( "1", name = "dep1" )
node = self._rebuildNode( vfile, builder, [value1, node3], [dep], tmp_files )
dep = SimpleValue( "11", name = "dep1" )
node = self._rebuildNode( vfile, builder, [value1, node3], [dep], tmp_files )
node3 = self._rebuildNode( vfile, builder3, [value1], [], tmp_files )
node = self._rebuildNode( vfile, builder, [value1], [node3], tmp_files )
node3 = self._rebuildNode( vfile, builder3, [value2], [], tmp_files )
node = self._rebuildNode( vfile, builder, [value1], [node3], tmp_files )
node_tname = node.getTargetValues()[0].name
with open( node_tname, 'wb' ) as f:
f.write( b'333' )
f.flush()
FileChecksumValue( node_tname, use_cache = False )
node = self._rebuildNode( vfile, builder, [value1], [node3], tmp_files )
with open( node.getSideEffectValues()[0].name, 'wb' ) as f:
f.write( b'abc' )
f.flush()
FileChecksumValue( node.getSideEffectValues()[0].name, use_cache = False )
node = Node( builder, [value1] )
node.depends( [node3] )
node.initiate()
self.assertTrue( node.checkActual( vfile ) )
# node = self._rebuildNode( vfile, builder, [value1], [node3], tmp_files )
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove( tmp_file )
except OSError:
pass
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic, _cyclic_deps, node4,
node3)
def _testNoBuildSpeed( vfile, builder, source_values ):
for source in source_values:
node = Node( builder, _FileValueType( source ) )
node.initiate()
if not node.checkActual( vfile ):
raise AssertionError( "node is not actual" )
def test_node_speed(self):
try:
tmp_files = []
source_files = _generate_files(tmp_files, 4000, 50 * 1024)
# idep_files = _copy_files(tmp_files, source_files, 'h')
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
builder = TestSpeedBuilder("TestSpeedBuilder", "tmp", "h")
for source in source_files:
node = Node(builder, _FileValueType(source))
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
for tmp_file in node.get_targets():
tmp_files.append(tmp_file)
t = lambda: _test_no_build_speed(vfile,
builder,
source_files)
t = timeit.timeit(t, number=1)
finally:
vfile.close()
finally:
for tmp_file in tmp_files:
try:
os.remove(tmp_file)
except OSError:
pass
def test_node_value(self):
with Tempfile() as tmp:
vfile = EntitiesFile(tmp)
try:
value1 = SimpleEntity("http://aql.org/download1",
name="target_url1")
value2 = SimpleEntity("http://aql.org/download2",
name="target_url2")
value3 = SimpleEntity("http://aql.org/download3",
name="target_url3")
dep_value1 = SimpleEntity("SOME_VALUE1", name="dep_value")
dep_value2 = SimpleEntity("SOME_VALUE2", name="dep_value")
options = builtin_options()
builder = ChecksumBuilder(options)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value1)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.initiate()
node.build_split(vfile, False)
self.assertTrue(node.check_actual(vfile, False))
# -----------------------------------------------------------
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
node.build()
node.save(vfile)
node = Node(builder, [value1, value2, value3])
node.depends(dep_value2)
node.depends(NullEntity())
node.initiate()
node.build_split(vfile, False)
self.assertFalse(node.check_actual(vfile, False))
finally:
vfile.close()