def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def test_bm_skip_nodes_by_option(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
cond_node = Node(CondBuilder(options), False)
options.has_openmp = BoolOptionType(default=True)
options.has_openmp = cond_node
node = Node(ValueBuilder(options), None)
bm.add([node])
bm.build_if(options.has_openmp, node)
bm.depends(node, [cond_node])
_build(bm, jobs=4)
self.assertEqual(self.built_nodes, 1)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleValue( "http://aql.org/download1", name = "target_url1" )
value2 = SimpleValue( "http://aql.org/download2", name = "target_url2" )
value3 = SimpleValue( "http://aql.org/download3", name = "target_url3" )
options = builtinOptions()
builder = CopyValueBuilder( options )
node0 = Node( builder, value1 )
node1 = Node( builder, node0 )
node2 = Node( builder, node1 )
node3 = Node( builder, value2 )
node4 = Node( builder, value3 )
node5 = Node( builder, node4 )
node6 = Node( builder, node5 )
node6.depends( [node0, node1] )
bm.add( node0 ); bm.selfTest(); self.assertEqual( len(bm), 1 )
bm.add( node1 ); bm.selfTest(); self.assertEqual( len(bm), 2 )
bm.add( node2 ); bm.selfTest(); self.assertEqual( len(bm), 3 )
bm.add( node3 ); bm.selfTest(); self.assertEqual( len(bm), 4 )
bm.add( node4 ); bm.selfTest(); self.assertEqual( len(bm), 5 )
bm.add( node5 ); bm.selfTest(); self.assertEqual( len(bm), 6 )
bm.add( node6 ); bm.selfTest(); self.assertEqual( len(bm), 7 )
node0.depends( node3 ); bm.depends( node0, node3 ); bm.selfTest()
node1.depends( node3 ); bm.depends( node1, node3 ); bm.selfTest()
node2.depends( node3 ); bm.depends( node2, node3 ); bm.selfTest()
node3.depends( node4 ); bm.depends( node3, node4 ); bm.selfTest()
node0.depends( node5 ); bm.depends( node0, node5 ); bm.selfTest()
node5.depends( node3 ); bm.depends( node5, node3 ); bm.selfTest()
with self.assertRaises(ErrorNodeDependencyCyclic):
node4.depends( node3 ); bm.depends( node4, node3 ); bm.selfTest()
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic,
_cyclic_deps, node4, node3)
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr,))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
for node in to_sequence(nodes):
node.depends(dependencies)
self.build_manager.depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test aq_test_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node,))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
def test_bm_deps(self):
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1", name="target_url1")
value2 = SimpleEntity("http://aql.org/download2", name="target_url2")
value3 = SimpleEntity("http://aql.org/download3", name="target_url3")
options = builtin_options()
builder = CopyValueBuilder(options)
node0 = Node(builder, value1)
node1 = Node(builder, node0)
node2 = Node(builder, node1)
node3 = Node(builder, value2)
node4 = Node(builder, value3)
node5 = Node(builder, node4)
node6 = Node(builder, node5)
node6.depends([node0, node1])
bm.add([node0])
bm.self_test()
self.assertEqual(len(bm), 1)
bm.add([node1])
bm.self_test()
self.assertEqual(len(bm), 2)
bm.add([node2])
bm.self_test()
self.assertEqual(len(bm), 3)
bm.add([node3])
bm.self_test()
self.assertEqual(len(bm), 4)
bm.add([node4])
bm.self_test()
self.assertEqual(len(bm), 5)
bm.add([node5])
bm.self_test()
self.assertEqual(len(bm), 6)
bm.add([node6])
bm.self_test()
self.assertEqual(len(bm), 7)
node0.depends(node3)
bm.depends(node0, [node3])
bm.self_test()
node1.depends(node3)
bm.depends(node1, [node3])
bm.self_test()
node2.depends(node3)
bm.depends(node2, [node3])
bm.self_test()
node3.depends(node4)
bm.depends(node3, [node4])
bm.self_test()
node0.depends(node5)
bm.depends(node0, [node5])
bm.self_test()
node5.depends(node3)
bm.depends(node5, [node3])
bm.self_test()
def _cyclic_deps(src_node, dep_node):
src_node.depends(dep_node)
bm.depends(src_node, [dep_node])
self.assertRaises(ErrorNodeDependencyCyclic, _cyclic_deps, node4,
node3)