def _build_nodes(num_dups, uptodate):
bm = BuildManager()
self.building_nodes = self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
src_entities = []
for s in src_files:
src_entities.append(FileChecksumEntity(s))
for i in range(num_dups):
num_built_nodes = 1
node = Node(builder, src_entities)
node = Node(builder, node)
num_built_nodes += 2
node = Node(builder, node)
num_built_nodes += 2**2
node = Node(builder, node)
num_built_nodes += 2**3
bm.add([node])
_build(bm, jobs=10, explain=False)
if uptodate:
num_built_nodes = 0
else:
num_built_nodes *= num_src_files
self.assertEqual(self.building_nodes, num_built_nodes)
def test_bm_no_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 256)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
node1 = Node(builder1, node1)
node2 = Node(builder2, node2)
bm.add([node1, node2])
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files * 2)
def test_bm_node_index(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 2
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
node = Node(builder, src_files)
nodes = [
Node(builder, node[i * 2]) for i in range(num_src_files + 1)
]
node2 = Node(builder, node[1:2][:])
#
bm.add([node2])
bm.add(nodes)
_build(bm)
self.assertEqual(self.built_nodes,
num_src_files + num_src_files + 1 + 1)
def test_bm_node_index(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 2
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
node = Node(builder, src_files)
nodes = [Node(builder, node[i * 2])
for i in range(num_src_files + 1)]
node2 = Node(builder, node[1:2][:])
#
bm.add([node2])
bm.add(nodes)
_build(bm)
self.assertEqual(
self.built_nodes, num_src_files + num_src_files + 1 + 1)
def test_bm_no_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 256)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
node1 = Node(builder1, node1)
node2 = Node(builder2, node2)
bm.add([node1, node2])
_build(bm)
self.assertEqual(
self.built_nodes, num_src_files + num_src_files * 2)
def _build_nodes(num_dups, uptodate):
bm = BuildManager()
self.building_nodes = self.built_nodes = 0
builder = ChecksumSingleBuilder(options, 0, 256)
src_entities = tuple(map(FileChecksumEntity, src_files))
num_built_nodes = 0
for i in range(num_dups):
num_built_nodes = 1
node = Node(builder, src_entities)
node = Node(builder, node)
num_built_nodes += 2
node = Node(builder, node)
num_built_nodes += 2**2
node = Node(builder, node)
num_built_nodes += 2**3
bm.add([node])
_build(bm, jobs=10, explain=False)
if uptodate:
num_built_nodes = 0
else:
num_built_nodes *= num_src_files
self.assertEqual(self.building_nodes, num_built_nodes)
def test_bm_node_names(self):
with Tempdir() as tmp_dir:
src_files = self.generate_source_files(tmp_dir, 3, 201)
options = builtin_options()
options.build_dir = tmp_dir
builder = ChecksumBuilder(options, 0, 256, replace_ext=False)
bm = BuildManager()
try:
src_entities = []
for s in src_files:
src_entities.append(FileChecksumEntity(s))
node0 = Node(builder, None)
node1 = Node(builder, src_entities)
node2 = Node(builder, node1)
node3 = Node(builder, node2)
node4 = Node(builder, node3)
bm.add([node0, node1, node2, node3, node4])
bm.build(1, False)
print("node2: %s" % str(node4))
print("node2: %s" % str(node3))
print("node2: %s" % str(node2))
print("node1: %s" % str(node1))
print("node0: %s" % str(node0))
finally:
bm.close()
def test_bm_sync_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(SyncValueBuilder(options, name="31", number=0,
sleep_interval=0),
SimpleEntity("31"))
node32 = Node(SyncValueBuilder(options, name="32", number=0,
sleep_interval=0),
SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31,))
node22 = Node(SyncValueBuilder(options, name="22", number=0,
sleep_interval=5),
(node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33,))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33,))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
bm.add((node10, node11))
bm.sync((node10, node11), deep=True)
_build(bm, jobs=4)
def test_bm_deps_speed(self):
bm = BuildManager()
value = SimpleEntity("http://aql.org/download", name="target_url1")
builder = CopyValueBuilder()
node = Node(builder, value)
bm.add([node])
_generate_node_tree(bm, builder, node, 5000)
def test_bm_deps_speed(self):
bm = BuildManager()
value = SimpleEntity("http://aql.org/download", name="target_url1")
builder = CopyValueBuilder()
node = Node(builder, value)
bm.add([node])
_generate_node_tree(bm, builder, node, 5000)
def _addNodesToBM( builder, src_files, Node = Node ):
bm = BuildManager()
try:
checksums_node = Node( builder, src_files )
checksums_node2 = Node( builder, checksums_node )
bm.add( checksums_node ); bm.selfTest()
bm.add( checksums_node2 ); bm.selfTest()
except Exception:
bm.close()
raise
return bm
def _add_nodes_to_bm(builder, src_files):
bm = BuildManager()
try:
checksums_node = Node(builder, src_files)
checksums_node2 = Node(builder, checksums_node)
bm.add([checksums_node])
bm.self_test()
bm.add([checksums_node2])
bm.self_test()
except Exception:
bm.close()
raise
return bm
def _add_nodes_to_bm(builder, src_files):
bm = BuildManager()
try:
checksums_node = Node(builder, src_files)
checksums_node2 = Node(builder, checksums_node)
bm.add([checksums_node])
bm.self_test()
bm.add([checksums_node2])
bm.self_test()
except Exception:
bm.close()
raise
return bm
def test_bm_skip_nodes_by_value(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
node = Node(ValueBuilder(options), [1, 2, 3, 4])
bm.add([node])
bm.build_if(False, node)
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 0)
def test_bm_skip_nodes_by_value(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
node = Node(ValueBuilder(options), [1, 2, 3, 4])
bm.add([node])
bm.build_if(False, node)
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 0)
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumBuilder(options, 0, 256)
options.batch_build = False
single_builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files + 1)
# -----------------------------------------------------------
self.regenerate_file(src_files[0], 201)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_install(self):
with Tempdir() as tmp_install_dir:
with Tempdir() as tmp_dir:
# tmp_install_dir = Tempdir()
# tmp_dir = Tempdir()
build_dir = os.path.join( tmp_dir, 'output' )
options = builtinOptions()
options.build_dir = build_dir
num_sources = 3
sources = self.generateSourceFiles( tmp_dir, num_sources, 200 )
installer = InstallBuilder( options, str(tmp_install_dir) )
bm = BuildManager()
try:
result = Node( installer, sources )
bm.add( result )
self._build( bm, jobs = 1, keep_going = False )
self.assertEqual( self.building_started, 1 )
self.assertEqual( self.building_started, self.building_finished )
bm.close()
result = Node( installer, sources )
bm = BuildManager()
bm.add( result )
self.building_started = 0
self._build( bm, jobs = 1, keep_going = False )
self.assertEqual( self.building_started, 0 )
finally:
bm.close()
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
options.batch_build = True
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumBuilder(options, 0, 256)
options.batch_build = False
single_builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files + 1)
# -----------------------------------------------------------
self.touch_cpp_file(src_files[0])
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(single_builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_bm_node_build_fail(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
builder = FailedBuilder(options)
nodes = [Node(builder, SimpleEntity("123-%s" % (i,)))
for i in range(4)]
bm.add(nodes)
self.assertRaises(Exception, _build, bm)
self.assertEqual(self.built_nodes, 0)
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
def test_bm_expensive(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
event = threading.Event()
heavy = ExpensiveValueBuilder(options, event, do_expensive=True)
light = ExpensiveValueBuilder(options, event, do_expensive=False)
node1 = Node(heavy, [1, 2, 3, 4, 5, 7])
node2 = Node(light, list(range(10, 100, 10)))
bm.add([node2, node1])
bm.expensive(node1)
_build(bm, jobs=16)
def test_bm_node_build_fail(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
builder = FailedBuilder(options)
nodes = [
Node(builder, SimpleEntity("123-%s" % (i, ))) for i in range(4)
]
bm.add(nodes)
self.assertRaises(Exception, _build, bm)
self.assertEqual(self.built_nodes, 0)
def test_bm_expensive(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
event = threading.Event()
heavy = ExpensiveValueBuilder(options, event, do_expensive=True)
light = ExpensiveValueBuilder(options, event, do_expensive=False)
node1 = Node(heavy, [1, 2, 3, 4, 5, 7])
node2 = Node(light, list(range(10, 100, 10)))
bm.add([node2, node1])
bm.expensive(node1)
_build(bm, jobs=16)
def test_bm_tags(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
builder = ChecksumSingleBuilder(options, 0, 256)
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, num_src_files * 2)
# -----------------------------------------------------------
self.regenerate_file(src_files[0], )
bm = BuildManager()
self.built_nodes = 0
node = Node(builder, src_files)
node_md5 = Node(builder, node.at('md5'))
bm.add([node_md5])
_build(bm)
self.assertEqual(self.built_nodes, 2)
def test_bm_tags_batch(self):
with Tempdir() as tmp_dir:
options = builtinOptions()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generateSourceFiles( tmp_dir, num_src_files, 201 )
builder = ChecksumBuilder( options, 0, 256 )
single_builder = ChecksumSingleBuilder( options, 0, 256 )
bm = BuildManager()
self.finished_nodes = 0
node = BatchNode( builder, src_files )
node_md5 = Node( single_builder, node.at('md5') )
bm.add( node_md5 )
_build( bm )
self.assertEqual( self.finished_nodes, num_src_files + 1 )
#//-------------------------------------------------------//
self.touchCppFile( src_files[0] )
bm = BuildManager()
self.finished_nodes = 0
node = BatchNode( builder, src_files )
node_md5 = Node( single_builder, node.at('md5') )
bm.add( node_md5 )
_build( bm )
self.assertEqual( self.finished_nodes, 2 )
def test_bm_rebuild(self):
with Tempdir() as tmp_dir:
options = builtinOptions()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generateSourceFiles( tmp_dir, num_src_files, 201 )
bm = BuildManager()
self.building_nodes = self.finished_nodes = 0
self.actual_nodes = self.outdated_nodes = 0
builder = ChecksumSingleBuilder( options, 0, 256 )
src_values = []
for s in src_files:
src_values.append( FileChecksumValue( s ) )
node = Node( builder, src_values )
node = Node( builder, node )
node = Node( builder, node )
bm.add( node )
_build( bm )
self.assertEqual( self.building_nodes, num_src_files * 7 )
#//-------------------------------------------------------//
self.actual_nodes = self.outdated_nodes = 0
bm = BuildManager()
builder = ChecksumSingleBuilder( options, 0, 256 )
node = Node( builder, src_values )
bm.add( node ); bm.selfTest()
bm.status(); bm.selfTest()
self.assertEqual( self.outdated_nodes, 0 )
self.assertEqual( self.actual_nodes, num_src_files )
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [
Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])
]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def test_exec(self):
with Tempdir() as tmp_dir:
build_dir = FilePath(tmp_dir).join('build')
options = builtinOptions()
cmd = [ whereProgram( "python" ), '-c', 'print("TEST EXEC")']
options.build_dir = build_dir
exec_cmd = ExecuteCommand( options )
bm = BuildManager()
try:
result = Node( exec_cmd, cmd )
bm.add( result )
self._build( bm, jobs = 1, keep_going = False )
self.assertEqual( self.building_started, 1 )
self.assertEqual( self.building_started, self.building_finished )
bm.close()
result = Node( exec_cmd, cmd )
bm = BuildManager()
bm.add( result )
self.building_started = 0
self._build( bm, jobs = 1, keep_going = False )
self.assertEqual( self.building_started, 0 )
finally:
bm.close()
def test_exec(self):
with Tempdir() as tmp_dir:
build_dir = os.path.join(tmp_dir, 'build')
options = builtin_options()
cmd = [sys.executable, '-c', 'print("TEST EXEC")']
options.build_dir = build_dir
exec_cmd = ExecuteCommandBuilder(options)
bm = BuildManager()
try:
result = Node(exec_cmd, cmd)
bm.add([result])
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 1)
self.assertEqual(self.building_started, self.built_nodes)
bm.close()
result = Node(exec_cmd, cmd)
bm = BuildManager()
bm.add([result])
self.building_started = 0
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 0)
finally:
bm.close()
def test_exec(self):
with Tempdir() as tmp_dir:
build_dir = os.path.join(tmp_dir, 'build')
options = builtin_options()
cmd = [sys.executable, '-c', 'print("TEST EXEC")']
options.build_dir = build_dir
exec_cmd = ExecuteCommandBuilder(options)
bm = BuildManager()
try:
result = Node(exec_cmd, cmd)
bm.add([result])
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 1)
self.assertEqual(self.building_started, self.built_nodes)
bm.close()
result = Node(exec_cmd, cmd)
bm = BuildManager()
bm.add([result])
self.building_started = 0
self._build(bm, jobs=1, keep_going=False)
self.assertEqual(self.building_started, 0)
finally:
bm.close()
def test_bm_skip_nodes_by_node(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node1 = Node(CondBuilder(options), False)
node1 = Node(ValueBuilder(options), [1, 2])
cond_node2 = Node(CondBuilder(options), True)
node2 = Node(ValueBuilder(options), [3, 4])
main = Node(ValueBuilder(options), [7, 8, node1, node2])
bm.add([main])
bm.build_if(cond_node1, node1)
bm.build_if(cond_node2, node2)
_build(bm, jobs=4)
self.assertEqual(main.get(), "7-8-3-4")
self.assertEqual(self.built_nodes, 4)
def test_bm_skip_nodes_by_node(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node1 = Node(CondBuilder(options), False)
node1 = Node(ValueBuilder(options), [1, 2])
cond_node2 = Node(CondBuilder(options), True)
node2 = Node(ValueBuilder(options), [3, 4])
main = Node(ValueBuilder(options), [7, 8, node1, node2])
bm.add([main])
bm.build_if(cond_node1, node1)
bm.build_if(cond_node2, node2)
_build(bm, jobs=4)
self.assertEqual(main.get(), "7-8-3-4")
self.assertEqual(self.built_nodes, 4)
def test_bm_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 1024)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
# node1 = Node( builder1, node1 )
# node2 = Node( builder2, node2 )
bm.add([node1, node2])
self.assertRaises(ErrorNodeSignatureDifferent, _build, bm)
def test_bm_conflicts(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
num_src_files = 3
src_files = self.generate_source_files(tmp_dir, num_src_files, 201)
bm = BuildManager()
self.built_nodes = 0
builder1 = ChecksumSingleBuilder(options, 0, 256)
builder2 = ChecksumSingleBuilder(options, 0, 1024)
node1 = Node(builder1, src_files)
node2 = Node(builder2, src_files)
# node1 = Node( builder1, node1 )
# node2 = Node( builder2, node2 )
bm.add([node1, node2])
self.assertRaises(ErrorNodeSignatureDifferent, _build, bm)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleValue( "http://aql.org/download1", name = "target_url1" )
value2 = SimpleValue( "http://aql.org/download2", name = "target_url2" )
value3 = SimpleValue( "http://aql.org/download3", name = "target_url3" )
options = builtinOptions()
builder = CopyValueBuilder( options )
node0 = Node( builder, value1 )
node1 = Node( builder, node0 )
node2 = Node( builder, node1 )
node3 = Node( builder, value2 )
node4 = Node( builder, value3 )
node5 = Node( builder, node4 )
node6 = Node( builder, node5 )
node6.depends( [node0, node1] )
bm.add( node0 ); bm.selfTest(); self.assertEqual( len(bm), 1 )
bm.add( node1 ); bm.selfTest(); self.assertEqual( len(bm), 2 )
bm.add( node2 ); bm.selfTest(); self.assertEqual( len(bm), 3 )
bm.add( node3 ); bm.selfTest(); self.assertEqual( len(bm), 4 )
bm.add( node4 ); bm.selfTest(); self.assertEqual( len(bm), 5 )
bm.add( node5 ); bm.selfTest(); self.assertEqual( len(bm), 6 )
bm.add( node6 ); bm.selfTest(); self.assertEqual( len(bm), 7 )
node0.depends( node3 ); bm.depends( node0, node3 ); bm.selfTest()
node1.depends( node3 ); bm.depends( node1, node3 ); bm.selfTest()
node2.depends( node3 ); bm.depends( node2, node3 ); bm.selfTest()
node3.depends( node4 ); bm.depends( node3, node4 ); bm.selfTest()
node0.depends( node5 ); bm.depends( node0, node5 ); bm.selfTest()
node5.depends( node3 ); bm.depends( node5, node3 ); bm.selfTest()
with self.assertRaises(ErrorNodeDependencyCyclic):
node4.depends( node3 ); bm.depends( node4, node3 ); bm.selfTest()
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr, ))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node, ))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
def test_bm_require_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(
SyncValueBuilder(options,
name="31",
number=0,
sleep_interval=0), SimpleEntity("31"))
node32 = Node(
SyncValueBuilder(options,
name="32",
number=0,
sleep_interval=0), SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31, ))
node22 = Node(
SyncValueBuilder(options,
name="22",
number=0,
sleep_interval=5), (node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33, ))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33, ))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
# print( "node30: %s" % node30 )
# print( "node31: %s" % node31 )
# print( "node32: %s" % node32 )
# print( "node33: %s" % node33 )
#
# print( "node20: %s" % node20 )
# print( "node21: %s" % node21 )
# print( "node22: %s" % node22 )
# print( "node23: %s" % node23 )
# print( "node24: %s" % node24 )
#
# print( "node10: %s" % node10 )
# print( "node11: %s" % node11 )
bm.add((node10, node11))
bm.module_depends(node10, [node11])
_build(bm, jobs=4)
def test_bm_require_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(SyncValueBuilder(options, name="31", number=0,
sleep_interval=0),
SimpleEntity("31"))
node32 = Node(SyncValueBuilder(options, name="32", number=0,
sleep_interval=0),
SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31,))
node22 = Node(SyncValueBuilder(options, name="22", number=0,
sleep_interval=5),
(node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33,))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33,))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
# print( "node30: %s" % node30 )
# print( "node31: %s" % node31 )
# print( "node32: %s" % node32 )
# print( "node33: %s" % node33 )
#
# print( "node20: %s" % node20 )
# print( "node21: %s" % node21 )
# print( "node22: %s" % node22 )
# print( "node23: %s" % node23 )
# print( "node24: %s" % node24 )
#
# print( "node10: %s" % node10 )
# print( "node11: %s" % node11 )
bm.add((node10, node11))
bm.module_depends(node10, [node11])
_build(bm, jobs=4)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic, _cyclic_deps, node4,
node3)
def test_bm_sync_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(
SyncValueBuilder(options,
name="31",
number=0,
sleep_interval=0), SimpleEntity("31"))
node32 = Node(
SyncValueBuilder(options,
name="32",
number=0,
sleep_interval=0), SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31, ))
node22 = Node(
SyncValueBuilder(options,
name="22",
number=0,
sleep_interval=5), (node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33, ))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33, ))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
bm.add((node10, node11))
bm.sync((node10, node11), deep=True)
_build(bm, jobs=4)
def test_bm_nodes(self):
def _make_nodes(builder):
node1 = Node(builder, value1)
copy_node1 = Node(builder, node1)
copy2_node1 = Node(builder, copy_node1)
node2 = Node(builder, value2)
node3 = Node(builder, value3)
copy_node3 = Node(builder, node3)
copy2_node3 = Node(builder, copy_node3)
copy2_node3.depends([node1, copy_node1])
return node1, node2, node3, copy_node1,\
copy_node3, copy2_node1, copy2_node3
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1",
name="target_url1")
value2 = SimpleEntity("http://aql.org/download2",
name="target_url2")
value3 = SimpleEntity("http://aql.org/download3",
name="target_url3")
builder = CopyValueBuilder(options)
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 7)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 0)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.removed_nodes = 0
bm.clear()
bm.close()
self.assertEqual(self.removed_nodes, 7)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 2)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
node2 = nodes[1]
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 1)
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr,))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node,))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
def test_bm_nodes(self):
def _make_nodes(builder):
node1 = Node(builder, value1)
copy_node1 = Node(builder, node1)
copy2_node1 = Node(builder, copy_node1)
node2 = Node(builder, value2)
node3 = Node(builder, value3)
copy_node3 = Node(builder, node3)
copy2_node3 = Node(builder, copy_node3)
copy2_node3.depends([node1, copy_node1])
return node1, node2, node3, copy_node1,\
copy_node3, copy2_node1, copy2_node3
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
value1 = SimpleEntity(
"http://aql.org/download1", name="target_url1")
value2 = SimpleEntity(
"http://aql.org/download2", name="target_url2")
value3 = SimpleEntity(
"http://aql.org/download3", name="target_url3")
builder = CopyValueBuilder(options)
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 7)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 0)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.removed_nodes = 0
bm.clear()
bm.close()
self.assertEqual(self.removed_nodes, 7)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 2)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
node2 = nodes[1]
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 1)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic,
_cyclic_deps, node4, node3)