def test_bm_sync_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(SyncValueBuilder(options, name="31", number=0,
sleep_interval=0),
SimpleEntity("31"))
node32 = Node(SyncValueBuilder(options, name="32", number=0,
sleep_interval=0),
SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31,))
node22 = Node(SyncValueBuilder(options, name="22", number=0,
sleep_interval=5),
(node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33,))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33,))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
bm.add((node10, node11))
bm.sync((node10, node11), deep=True)
_build(bm, jobs=4)
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
def test_bm_sync_nodes(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
nodes = [
Node(SyncValueBuilder(options, name="%s" % i, number=n),
SimpleEntity("123-%s" % i))
for i, n in zip(range(4), [3, 5, 7, 11])
]
bm.add(nodes)
bm.sync(nodes)
_build(bm, jobs=4)
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr, ))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(dep for dep in to_sequence(dependencies)
if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node, ))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
class Project(object):
def __init__(self, config):
self.targets = config.targets
self.options = config.options
self.arguments = config.arguments
self.config = config
self.scripts_cache = {}
self.configs_cache = {}
self.aliases = {}
self.alias_descriptions = {}
self.defaults = []
self.build_manager = BuildManager()
self.tools = ProjectTools(self)
# -----------------------------------------------------------
def __getattr__(self, attr):
if attr == 'script_locals':
self.script_locals = self.__get_script_locals()
return self.script_locals
raise AttributeError("No attribute '%s'" % (attr,))
# -----------------------------------------------------------
def __get_script_locals(self):
script_locals = {
'options': self.options,
'tools': self.tools,
'Tool': self.tools.get_tool,
'TryTool': self.tools.try_tool,
'Tools': self.tools.get_tools,
'AddTool': self.tools.add_tool,
'LoadTools': self.tools.tools.load_tools,
'FindFiles': find_files,
'GetProject': self.get_project,
'GetProjectConfig': self.get_project_config,
'GetBuildTargets': self.get_build_targets,
'File': self.make_file_entity,
'Entity': self.make_entity,
'Dir': self.make_dir_entity,
'Config': self.read_config,
'Script': self.read_script,
'SetBuildDir': self.set_build_dir,
'Depends': self.depends,
'Requires': self.requires,
'RequireModules': self.require_modules,
'Sync': self.sync_nodes,
'BuildIf': self.build_if,
'SkipIf': self.skip_if,
'Alias': self.alias_nodes,
'Default': self.default_build,
'AlwaysBuild': self.always_build,
'Expensive': self.expensive,
'Build': self.build,
'Clear': self.clear,
'DirName': self.node_dirname,
'BaseName': self.node_basename,
}
return script_locals
# -----------------------------------------------------------
def get_project(self):
return self
# -----------------------------------------------------------
def get_project_config(self):
return self.config
# -----------------------------------------------------------
def get_build_targets(self):
return self.targets
# -----------------------------------------------------------
def make_file_entity(self, filepath, options=None):
if options is None:
options = self.options
file_type = FileTimestampEntity \
if options.file_signature == 'timestamp' \
else FileChecksumEntity
return file_type(filepath)
# -----------------------------------------------------------
def make_dir_entity(self, filepath):
return DirEntity(filepath)
# -----------------------------------------------------------
def make_entity(self, data, name=None):
return SimpleEntity(data=data, name=name)
# -----------------------------------------------------------
def _get_config_options(self, config, options):
if options is None:
options = self.options
options_ref = options.get_hash_ref()
config = os.path.normcase(os.path.abspath(config))
options_set = self.configs_cache.setdefault(config, set())
if options_ref in options_set:
return None
options_set.add(options_ref)
return options
# -----------------------------------------------------------
def _remove_overridden_options(self, result):
for arg in self.arguments:
try:
del result[arg]
except KeyError:
pass
# -----------------------------------------------------------
def read_config(self, config, options=None):
options = self._get_config_options(config, options)
if options is None:
return
config_locals = {'options': options}
dir_name, file_name = os.path.split(config)
with Chdir(dir_name):
result = exec_file(file_name, config_locals)
tools_path = result.pop('tools_path', None)
if tools_path:
self.tools.tools.load_tools(tools_path)
self._remove_overridden_options(result)
options.update(result)
# -----------------------------------------------------------
def read_script(self, script):
script = os.path.normcase(os.path.abspath(script))
scripts_cache = self.scripts_cache
script_result = scripts_cache.get(script, None)
if script_result is not None:
return script_result
dir_name, file_name = os.path.split(script)
with Chdir(dir_name):
script_result = exec_file(file_name, self.script_locals)
scripts_cache[script] = script_result
return script_result
# -----------------------------------------------------------
def add_nodes(self, nodes):
self.build_manager.add(nodes)
# -----------------------------------------------------------
def set_build_dir(self, build_dir):
build_dir = os.path.abspath(expand_file_path(build_dir))
if self.options.build_dir != build_dir:
self.options.build_dir = build_dir
# -----------------------------------------------------------
def build_if(self, condition, nodes):
self.build_manager.build_if(condition, nodes)
# -----------------------------------------------------------
def skip_if(self, condition, nodes):
self.build_manager.skip_if(condition, nodes)
# -----------------------------------------------------------
def depends(self, nodes, dependencies):
dependencies = tuple(to_sequence(dependencies))
depends = self.build_manager.depends
for node in to_sequence(nodes):
node.depends(dependencies)
depends(node, node.dep_nodes)
# -----------------------------------------------------------
def requires(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
depends = self.build_manager.depends
for node in to_sequence(nodes):
depends(node, dependencies)
# -----------------------------------------------------------
def require_modules(self, nodes, dependencies):
dependencies = tuple(
dep for dep in to_sequence(dependencies) if isinstance(dep, Node))
module_depends = self.build_manager.module_depends
for node in to_sequence(nodes):
module_depends(node, dependencies)
# -----------------------------------------------------------
# TODO: It works not fully correctly yet. See test test_bm_sync_modules
# def SyncModules( self, nodes ):
# nodes = tuple( node for node in to_sequence( nodes )
# if isinstance( node, Node ) )
# self.build_manager.sync( nodes, deep = True)
# -----------------------------------------------------------
def sync_nodes(self, *nodes):
nodes = flatten_list(nodes)
nodes = tuple(node for node in nodes if isinstance(node, Node))
self.build_manager.sync(nodes)
# -----------------------------------------------------------
def alias_nodes(self, alias, nodes, description=None):
for alias, node in itertools.product(to_sequence(alias),
to_sequence(nodes)):
self.aliases.setdefault(alias, set()).add(node)
if description:
self.alias_descriptions[alias] = description
# -----------------------------------------------------------
def default_build(self, nodes):
for node in to_sequence(nodes):
self.defaults.append(node)
# -----------------------------------------------------------
def always_build(self, nodes):
null_value = NullEntity()
for node in to_sequence(nodes):
node.depends(null_value)
# ----------------------------------------------------------
def expensive(self, nodes):
self.build_manager.expensive(nodes)
# ----------------------------------------------------------
def _add_alias_nodes(self, target_nodes, aliases):
try:
for alias in aliases:
target_nodes.update(self.aliases[alias])
except KeyError as ex:
raise ErrorProjectUnknownTarget(ex.args[0])
# ----------------------------------------------------------
def _add_default_nodes(self, target_nodes):
for node in self.defaults:
if isinstance(node, Node):
target_nodes.add(node)
else:
self._add_alias_nodes(target_nodes, (node,))
# ----------------------------------------------------------
def _get_build_nodes(self):
target_nodes = set()
self._add_alias_nodes(target_nodes, self.targets)
if not target_nodes:
self._add_default_nodes(target_nodes)
if not target_nodes:
target_nodes = None
return target_nodes
# ----------------------------------------------------------
def _get_jobs_count(self, jobs=None):
if jobs is None:
jobs = self.config.jobs
if not jobs:
jobs = 0
else:
jobs = int(jobs)
if not jobs:
jobs = cpu_count()
if jobs < 1:
jobs = 1
elif jobs > 32:
jobs = 32
return jobs
# ----------------------------------------------------------
def build(self, jobs=None):
jobs = self._get_jobs_count(jobs)
if not self.options.batch_groups.is_set():
self.options.batch_groups = jobs
build_nodes = self._get_build_nodes()
config = self.config
keep_going = config.keep_going,
explain = config.debug_explain
with_backtrace = config.debug_backtrace
force_lock = config.force_lock
use_sqlite = config.use_sqlite
is_ok = self.build_manager.build(jobs=jobs,
keep_going=bool(keep_going),
nodes=build_nodes,
explain=explain,
with_backtrace=with_backtrace,
use_sqlite=use_sqlite,
force_lock=force_lock)
return is_ok
# ----------------------------------------------------------
def clear(self):
build_nodes = self._get_build_nodes()
force_lock = self.config.force_lock
use_sqlite = self.config.use_sqlite
self.build_manager.clear(nodes=build_nodes,
use_sqlite=use_sqlite,
force_lock=force_lock)
# ----------------------------------------------------------
def list_targets(self):
targets = []
node2alias = {}
for alias, nodes in self.aliases.items():
key = frozenset(nodes)
target_info = node2alias.setdefault(key, [[], ""])
target_info[0].append(alias)
description = self.alias_descriptions.get(alias, None)
if description:
if len(target_info[1]) < len(description):
target_info[1] = description
build_nodes = self._get_build_nodes()
self.build_manager.shrink(build_nodes)
build_nodes = self.build_manager.get_nodes()
for nodes, aliases_and_description in node2alias.items():
aliases, description = aliases_and_description
aliases.sort(key=str.lower)
max_alias = max(aliases, key=len)
aliases.remove(max_alias)
aliases.insert(0, max_alias)
is_built = (build_nodes is None) or nodes.issubset(build_nodes)
targets.append((tuple(aliases), is_built, description))
# sorted list in format: [(target_names, is_built, description), ...]
targets.sort(key=lambda names: names[0][0].lower())
return _text_targets(targets)
# ----------------------------------------------------------
def list_options(self, brief=False):
result = self.options.help_text("Builtin options:", brief=brief)
result.append("")
tool_names = self.tools._get_tool_names()
if tool_names:
result.append("Available options of tools: %s" %
(', '.join(tool_names)))
if result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def list_tools_options(self, tools, brief=False):
tools = set(to_sequence(tools))
result = []
for tools_options, names in self.tools._get_tools_options().items():
names_set = tools & set(names)
if names_set:
tools -= names_set
options_name = "Options of tool: %s" % (', '.join(names))
result += tools_options.help_text(options_name, brief=brief)
if result and result[-1]:
result.append("")
return result
# ----------------------------------------------------------
def node_dirname(self, node):
return NodeDirNameFilter(node)
# ----------------------------------------------------------
def node_basename(self, node):
return NodeBaseNameFilter(node)
def test_bm_sync_modules(self):
with Tempdir() as tmp_dir:
options = builtin_options()
options.build_dir = tmp_dir
bm = BuildManager()
self.built_nodes = 0
"""
10 11__
/ | \ / \ \
20 21 22 23 24
/ \ | / \ \ |
30 31 32 33
"""
node30 = Node(SyncValueBuilder(options, name="30", number=7),
SimpleEntity("30"))
node31 = Node(
SyncValueBuilder(options,
name="31",
number=0,
sleep_interval=0), SimpleEntity("31"))
node32 = Node(
SyncValueBuilder(options,
name="32",
number=0,
sleep_interval=0), SimpleEntity("32"))
node33 = Node(SyncValueBuilder(options, name="33", number=17),
SimpleEntity("33"))
node20 = Node(SyncValueBuilder(options, name="20", number=7),
(node30, node31))
node21 = Node(SyncValueBuilder(options, name="21", number=7),
(node31, ))
node22 = Node(
SyncValueBuilder(options,
name="22",
number=0,
sleep_interval=5), (node31, node32))
node23 = Node(SyncValueBuilder(options, name="23", number=17),
(node33, ))
node24 = Node(SyncValueBuilder(options, name="24", number=17),
(node33, ))
node10 = Node(SyncValueBuilder(options, name="10", number=7),
(node20, node21, node22))
node11 = Node(SyncValueBuilder(options, name="11", number=17),
(node22, node23, node24))
bm.add((node10, node11))
bm.sync((node10, node11), deep=True)
_build(bm, jobs=4)
