def test_bm_node_names(self):
with Tempdir() as tmp_dir:
src_files = self.generate_source_files(tmp_dir, 3, 201)
options = builtin_options()
options.build_dir = tmp_dir
builder = ChecksumBuilder(options, 0, 256, replace_ext=False)
bm = BuildManager()
try:
src_entities = []
for s in src_files:
src_entities.append(FileChecksumEntity(s))
node0 = Node(builder, None)
node1 = Node(builder, src_entities)
node2 = Node(builder, node1)
node3 = Node(builder, node2)
node4 = Node(builder, node3)
bm.add([node0, node1, node2, node3, node4])
bm.build(1, False)
print("node2: %s" % str(node4))
print("node2: %s" % str(node3))
print("node2: %s" % str(node2))
print("node1: %s" % str(node1))
print("node0: %s" % str(node0))
finally:
bm.close()
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr, ))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node, ))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr,))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node,))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
def test_bm_nodes(self):
def _make_nodes(builder):
node1 = Node(builder, value1)
copy_node1 = Node(builder, node1)
copy2_node1 = Node(builder, copy_node1)
node2 = Node(builder, value2)
node3 = Node(builder, value3)
copy_node3 = Node(builder, node3)
copy2_node3 = Node(builder, copy_node3)
copy2_node3.depends([node1, copy_node1])
return node1, node2, node3, copy_node1,\
copy_node3, copy2_node1, copy2_node3
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
value1 = SimpleEntity(
"http://aql.org/download1", name="target_url1")
value2 = SimpleEntity(
"http://aql.org/download2", name="target_url2")
value3 = SimpleEntity(
"http://aql.org/download3", name="target_url3")
builder = CopyValueBuilder(options)
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 7)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 0)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.removed_nodes = 0
bm.clear()
bm.close()
self.assertEqual(self.removed_nodes, 7)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 2)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
node2 = nodes[1]
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 1)
def test_bm_nodes(self):
def _make_nodes(builder):
node1 = Node(builder, value1)
copy_node1 = Node(builder, node1)
copy2_node1 = Node(builder, copy_node1)
node2 = Node(builder, value2)
node3 = Node(builder, value3)
copy_node3 = Node(builder, node3)
copy2_node3 = Node(builder, copy_node3)
copy2_node3.depends([node1, copy_node1])
return node1, node2, node3, copy_node1,\
copy_node3, copy2_node1, copy2_node3
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
value1 = SimpleEntity("http://aql.org/download1",
name="target_url1")
value2 = SimpleEntity("http://aql.org/download2",
name="target_url2")
value3 = SimpleEntity("http://aql.org/download3",
name="target_url3")
builder = CopyValueBuilder(options)
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 7)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.built_nodes = 0
bm.build(jobs=1, keep_going=False)
bm.close()
self.assertEqual(self.built_nodes, 0)
# ----------------------------------------------------------
bm.add(_make_nodes(builder))
self.removed_nodes = 0
bm.clear()
bm.close()
self.assertEqual(self.removed_nodes, 7)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 2)
# ----------------------------------------------------------
nodes = _make_nodes(builder)
node2 = nodes[1]
copy_node3 = nodes[4]
bm.add(nodes)
self.built_nodes = 0
bm.build(jobs=1, keep_going=False, nodes=[node2, copy_node3])
bm.close()
self.assertEqual(self.built_nodes, 1)