def __init__(self, funcs=None):
if isinstance(funcs, dict):
self.funcs = funcs
logger.info(
'created build generator object with a default job mapping.')
else:
self.funcs = {}
logger.info(
'created build generator object with an empty job mapping.')
self._stages = BuildSystem()
logger.info('created empty build system object for build generator.')
self._process_jobs = {}
"Mapping of targets to job tuples."
self._process_tree = {}
"Internal representation of the dependency tree."
self.specs = {}
"Mapping of job specs targets to specs."
self.system = None
self._final = False
"""Internal switch that indicates a finalized method. See
:meth:`~system.BuildSystemGenerator.finlize()` for more information."""
self.check = DependencyChecks()
logger.info('created build system generator object')
def __init__(self, funcs=None):
if isinstance(funcs, dict):
self.funcs = funcs
logger.info('created build generator object with a default job mapping.')
else:
self.funcs = {}
logger.info('created build generator object with an empty job mapping.')
self._stages = BuildSystem()
logger.info('created empty build system object for build generator.')
self._process_jobs = {}
"Mapping of targets to job tuples."
self._process_tree = {}
"Internal representation of the dependency tree."
self.specs = {}
"Mapping of job specs targets to specs."
self.system = None
self._final = False
"""Internal switch that indicates a finalized method. See
:meth:`~system.BuildSystemGenerator.finlize()` for more information."""
self.check = DependencyChecks()
logger.info('created build system generator object')
def setUp(self):
self.d = DependencyChecks()
self.fn_a = 'fn_a'
self.fn_b = 'fn_b'
class TestDependencyChecking(TestCase):
@classmethod
def setUp(self):
self.d = DependencyChecks()
self.fn_a = 'fn_a'
self.fn_b = 'fn_b'
@classmethod
def tearDown(self):
for fn in [ self.fn_a, self.fn_b ]:
if os.path.exists(fn):
os.remove(fn)
def ensure_clean(self):
self.assertFalse(os.path.exists(self.fn_a))
self.assertFalse(os.path.exists(self.fn_b))
def test_basic(self):
self.ensure_clean()
touch(self.fn_a)
self.assertTrue(os.path.exists(self.fn_a))
self.assertFalse(os.path.exists(self.fn_b))
def test_basic_alt(self):
self.ensure_clean()
touch(self.fn_b)
self.assertTrue(os.path.exists(self.fn_b))
self.assertFalse(os.path.exists(self.fn_a))
def test_default_method(self):
self.ensure_clean()
self.assertTrue(self.d.check_method, 'mtime')
def test_setting_valid_methods(self):
self.ensure_clean()
for method in ['force', 'ignore', 'hash', 'mtime']:
self.d.check_method = method
self.assertTrue(self.d.check_method, method)
self.d.check_method = 'mtime'
self.assertTrue(self.d.check_method, 'mtime')
def test_setting_invalid_method(self):
self.ensure_clean()
with self.assertRaises(DependencyCheckError):
self.d.check_method = 'magic'
def test_mtime_rebuild(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.assertTrue(self.d.check_method, 'mtime')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_mtime_no_rebuild(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.assertTrue(self.d.check_method, 'mtime')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_mtime_rebuild_no_target(self):
self.ensure_clean()
touch(self.fn_b)
self.assertTrue(self.d.check_method, 'mtime')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild(self):
self.ensure_clean()
write(self.fn_a, 'aaa')
write(self.fn_b, 'bbb')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild_ignoring_update_order(self):
self.ensure_clean()
write(self.fn_b, 'bbb')
breath()
breath()
write(self.fn_a, 'aaa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_no_rebuild(self):
self.ensure_clean()
write(self.fn_b, 'aaa')
write(self.fn_a, 'aaa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild_no_target(self):
self.ensure_clean()
write(self.fn_b, 'aa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_non_existing(self):
self.ensure_clean()
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_with_files(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_with_reversed_files(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_ignore_non_existing(self):
self.ensure_clean()
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_ignore_with_files(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_ignore_with_reversed_files(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
class BuildSystemGenerator(object):
"""
:class:`~system.BuildSystemGenerator` objects provide a unifying interface
for generating and running :class:`~system.BuildSystem` objects from
easy-to-edit sources. :class:`~system.BuildSystemGenerator` is the
fundamental basis of the :ref:`buildc` tool.
:param dict funcs: Optional. A dictionary that maps callable objects
(values), to identifiers used by build system definitions. If you do not
specify.
:class:`~system.BuildSystemGenerator` is designed to parse streams of YAML
documents that resemble the following:
.. code-block:: yaml
job: <func>
args: <<arg>,<list>>
stage: <str>
---
job: <func>
args: <<arg>,<list>>
target: <str>
dependency: <<str>,<list>>
---
dir: <<path>,<list>>
cmd: <str>
args: <<arg>,<list>>
stage: <str>
---
dir: <<path>,<list>>
cmd: <str>
args: <<arg>,<list>>
dependency: <<str>,<list>>
---
tasks:
- job: <func>
args: <<arg>,<list>>
- dir: <<path>,<list>>
cmd: <str>
args <<args>,<list>>
stage: <str>
...
See :doc:`/tutorial/use-buildc` for documentation of the input data
format and its use.
"""
def __init__(self, funcs=None):
if isinstance(funcs, dict):
self.funcs = funcs
logger.info(
'created build generator object with a default job mapping.')
else:
self.funcs = {}
logger.info(
'created build generator object with an empty job mapping.')
self._stages = BuildSystem()
logger.info('created empty build system object for build generator.')
self._process_jobs = {}
"Mapping of targets to job tuples."
self._process_tree = {}
"Internal representation of the dependency tree."
self.specs = {}
"Mapping of job specs targets to specs."
self.system = None
self._final = False
"""Internal switch that indicates a finalized method. See
:meth:`~system.BuildSystemGenerator.finlize()` for more information."""
self.check = DependencyChecks()
logger.info('created build system generator object')
@property
def check_method(self):
return self.check.check_method
@check_method.setter
def check_method(self, value):
if value not in self.check.checks:
pass
else:
self.check.check_method = value
def finalize(self):
"""
:raises: :exc:`~err.InvalidSystem` if you call
:meth:`~system.BuildSystemGenerator.finalize()` more than once on a
single :class:`~system.BuildSystemGenerator()` object.
You must call :meth:`~system.BuildSystemGenerator.finalize()` before
running the build system.
:meth:`~system.BuildSystemGenerator.finalize()` compiles the build
system. If there are no tasks with dependencies,
:attr:`~system.BuildSystemGenerator._stages` becomes the
:attr:`~system.BuildSystemGenerator.system` object. Otherwise,
:meth:`~system.BuildSystemGenerator.finalize()` orders the tasks with
dependencies, inserts them into a :class:`~stages.BuildSequence` object
before inserting the :attr:`~system.BuildSystemGenerator._stages` tasks.
"""
if self._final is False and self.system is None:
if len(self._process_tree) == 0:
logger.debug(
'no dependency tasks exist, trying to add build stages.')
if self._stages.count() > 0:
self.system = self._stages
logger.info(
'added tasks from build stages to build system.')
else:
logger.critical(
'cannot finalize empty generated BuildSystem object.')
raise InvalidSystem
elif len(self._process_tree) > 0:
self.system = BuildSystem(self._stages)
self._process = tsort(self._process_tree)
logger.debug('successfully sorted dependency tree.')
self._finalize_process_tree()
if self._stages.count() > 0:
self.system.extend(self._stages)
logger.info('added stages tasks to build system.')
self.system.close()
self._final = True
else:
logger.critical('cannot finalize object')
raise InvalidSystem
def _finalize_process_tree(self):
"""
Loops over the :attr:`~system.BuildSystemGenerator._process` list tree
created in the main :meth:`~system.BuildSystemGenerator.finalize()`
method, and adds tasks to :attr:`~system.BuildSystemGenerator.system`
object, which is itself a :class:`~system.BuildSystem` object.
While constructing the :attr:`~system.BuildSystemGenerator.system`
object, :meth:`~system.BuildSystemGenerator._finalize_process_tree()`
combines adjacent tasks that do not depend upon each other to increase
the potential for parallel execution.
:meth:`~system.BuildSystemGenerator._finalize_process`, calls
:meth:`~system.BuildSystemGenerator._add_tasks_to_stage()` which may
raise :exc:`~err.InvalidSystem` in the case of malformed tasks.
"""
total = len(self._process)
rebuilds_needed = False
idx = -1
stack = []
for i in self._process:
idx += 1
if rebuilds_needed is False:
if self._process_jobs[i][1] is False:
logger.debug(
'{0}: does not need a rebuild, passing'.format(i))
continue
elif self._process_jobs[i][1] is True:
logger.debug('{0}: needs rebuild.'.format(i))
rebuilds_needed = True
# rebuild needed here.
if idx + 1 == total:
# last task in tree.
if stack:
# if there are non dependent items in the stack, we
# might as well add the current task there.
stack.append(i)
else:
# if there's no stack, we'll just add the task directly.
pass
else:
if i not in self._process_tree[self._process[idx]]:
stack.append(i)
logger.debug(
'adding task {0} to queue not continuing.'.format(i))
continue
else:
# here the previous stack doesn't depend on the
# current task. add current task to stack then build
# the stack in parallel.
stack.append(i)
logger.debug(
'adding task to the stage queue, but not continuing'.
format(i))
self._add_tasks_to_stage(rebuilds_needed, idx, total, i, stack)
stack = []
def _add_tasks_to_stage(self, rebuilds_needed, idx, total, task, stack):
"""
:param bool rebuild_needed: ``True``, when the dependencies require a
rebuild of this target.
:param int idx: The index of the current task in the build dependency
chain.
:param int total: The total number of tasks in the dependency chain.
:param string task: The name of a task in the build process.
:param list stack: The list of non-dependent tasks that must be rebuilt,
not including the current task.
:raises: :exc:`~err.InvalidSystem` if its impossible to add a task in
the :class:`~system.BuildSystemGenerator` object to the finalized
build system object.
Called by :meth:`~system.BuildSystemGenerator._finalize_process_tree()`
to add and process tasks.
"""
if rebuilds_needed is True:
self.system.new_stage(task)
if stack:
for job in stack:
self.system.stages[task].add(self._process_jobs[job][0][0],
self._process_jobs[job][0][1])
else:
logger.debug('{0}: adding to rebuild queue'.format(task))
self.system.stages[task].add(self._process_jobs[task][0][0],
self._process_jobs[task][0][1])
elif rebuilds_needed is False:
logger.warning(
"dropping {0} task, no rebuild needed.".format(task))
return None
@staticmethod
def process_strings(spec, strings):
"""
:param string spec: A string, possibly with ``format()`` style tokens in curly braces.
:param dict strings: A mapping of strings to replacement values.
:raises: :exc:`python:TypeError` if ``strings`` is not a dict *and*
there may be a replacement string.
"""
if strings is None:
return spec
elif not isinstance(strings, dict):
logger.critical('replacement content must be a dictionary.')
raise TypeError
if isinstance(spec, dict):
out = {}
for k, v in spec.items():
if '{' in v and '}' in v:
try:
out[k] = v.format(**strings)
except KeyError as e:
msg = '{0} does not have {1} suitable replacement keys.'.format(
strings, e)
logger.critical(msg)
raise InvalidJob(msg)
else:
out[k] = v
return out
else:
if '{' in spec and '}' in spec:
try:
spec = spec.format(**strings)
except KeyError as e:
msg = '{0} does not have {1} suitable replacement keys.'.format(
spec, e)
logger.critical(msg)
raise InvalidJob(msg)
return spec
else:
return spec
@staticmethod
def generate_job(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names (i.e. ``spec.job`` to
functions.)
:returns: A tuple where the first item is a callable and the second item
is either a dict or a tuple of arguments, depending on the type of
the ``spec.args`` value.
:raises: :exc:`~err.InvalidStage` if ``spec.args`` is neither a
``dict``, ``list``, or ``tuple``.
"""
if isinstance(spec['args'], dict):
args = spec['args']
elif isinstance(spec['args'], list):
args = tuple(spec['args'])
elif isinstance(spec['args'], tuple):
args = spec['args']
else:
raise InvalidJob('args are malformed.')
if is_function(spec['job']):
action = spec['job']
else:
try:
action = funcs[spec['job']]
except KeyError:
msg = "{0} not in function specification with: {0} keys".format(
spec['job'], funcs.keys())
logger.critical(msg)
raise InvalidJob(msg)
return action, args
@staticmethod
def generate_shell_job(spec):
"""
:param dict spec: A *job* specification.
:returns: A tuple where the first element is :mod:`python:subprocess`
:func:`~python:subprocess.call`, and the second item is a dict
``args``, a ``dir``, and ``cmd`` keys.
Takes a ``spec`` dict and returns a tuple to define a task.
"""
if isinstance(spec['cmd'], list):
cmd_str = spec['cmd']
else:
cmd_str = spec['cmd'].split()
if isinstance(spec['dir'], list):
base_path = os.path.sep.join(spec['dir'])
if spec['dir'][0].startswith(os.path.sep):
spec['dir'] = base_path
else:
spec['dir'] = os.path.abspath(base_path)
if isinstance(spec['args'], list):
cmd_str.extend(spec['args'])
else:
cmd_str.extend(spec['args'].split())
return subprocess.call, dict(cwd=spec['dir'], args=cmd_str)
@staticmethod
def generate_sequence(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names to callable objects.
:returns: A :class:`~stages.BuildSequence()` object.
:raises: An exception when ``spec`` does not have a ``task`` element or
when ``task.spec`` is not a list.
Process a job spec that describes a sequence of tasks and returns a
:class:`~stages.BuildSequence()` object for inclusion in a
:class:`~system.BuildSystem()` object.
"""
sequence = BuildSequence()
if 'tasks' not in spec or not isinstance(spec['tasks'], list):
raise InvalidStage
else:
for task in spec['tasks']:
if 'job' in task:
job, args = BuildSystemGenerator.generate_job(task, funcs)
elif 'cmd' in task:
job, args = BuildSystemGenerator.generate_shell_job(task)
sequence.add(job, args)
sequence.close()
return sequence
def add_task(self, name, func):
"""
:param string name: The identifier of a callable.
:param callable func: A callable object.
:raises: :exc:`~err.InvaidJob` if ``func`` is not callable.
Adds a callable object to the :attr:`~system.BuildSystemGenerator.funcs`
attribute with the identifier ``name``.
"""
if not is_function(func):
logger.critical(
'cannot add tasks that are not callables ({0}).'.format(name))
raise InvalidJob("{0} is not callable.".format(func))
logger.debug('adding task named {0}'.format(name))
self.funcs[name] = func
def ingest(self, jobs, strings=None):
"""
:param iterable jobs: An interable object that contains build job
specifications.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
:returns: The number of jobs added to the build system.
For every document
"""
job_count = 0
for spec in jobs:
self._process_job(spec, strings)
job_count += 1
logger.debug('loaded {0} jobs'.format(job_count))
return job_count
def ingest_yaml(self, filename, strings=None):
"""
Wraps :meth:`~BuildSystemGenerator.ingest()`.
:param string filename: The fully qualified path name of a :term:`YAML`
file that contains a build system specification.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
For every document in the YAML file specified by ``filename``, calls
:meth:`~system.BuildSystemGenerator._process_job()` to add that job to
the :attr:`~system.BuildSystemGenerator.system` object.
"""
logger.debug('opening yaml file {0}'.format(filename))
try:
with open(filename, 'r') as f:
try:
jobs = yaml.safe_load_all(f)
except NameError:
msg = 'attempting to load a yaml definition without PyYAML installed.'
logger.critical(msg)
raise StageRunError(msg)
job_count = self.ingest(jobs, strings)
logger.debug('loaded {0} jobs from {1}'.format(
job_count, filename))
except IOError:
logger.warning('file {0} does not exist'.format(filename))
def ingest_json(self, filename, strings=None):
"""
Wraps :meth:`~BuildSystemGenerator.ingest()`.
:param string filename: The fully qualified path name of a :term:`JSON`
file that contains a build system specification.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
For every document in the JSON file specified by ``filename``, calls
:meth:`~system.BuildSystemGenerator._process_job()` to add that job to
the :attr:`~system.BuildSystemGenerator.system` object.
"""
logger.debug('opening json file {0}'.format(filename))
try:
with open(filename, 'r') as f:
jobs = json.load(f)
job_count = self.ingest(jobs, strings)
logger.debug('loaded {0} jobs from {1}'.format(
job_count, filename))
except IOError:
logger.warning('file {0} does not exist'.format(filename))
def _process_stage(self, spec, spec_keys=None, strings=None):
"""
:param dict spec: The task specification imported from user input.
:param set spec_keys: A set containing the top level keys in the set.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
:returns: A two item tuple, containing a function and a list of arguments.
:raises: :exc:`~err.InvalidJob` if the schema of ``spec`` is not recognized.
Based on the schema of the ``spec``, creates tasks and sequences to add
to a BuildSystem using, as appropriate one of the following methods:
- :meth:`~system.BuildSystemGenerator.generate_job()`,
- :meth:`~system.BuildSystemGenerator.generate_shell_job()`, or
- :meth:`~system.BuildSystemGenerator.generate_sequence()`.
"""
if spec_keys is None:
spec_keys = set(spec.keys())
elif isinstance(spec_keys, set):
logger.debug(
'using {0} for spec_keys as an optimization'.format(spec_keys))
else:
logger.error(
'spec_keys argument {0} must be a set'.format(spec_keys))
raise InvalidJob('problem with spec_keys')
if spec_keys.issuperset(set(['job', 'args'])):
logger.debug('spec looks like a pure python job, processing now.')
spec = self.process_strings(spec, strings)
return self.generate_job(spec, self.funcs)
elif spec_keys.issuperset(set([
'dir',
'cmd',
'args',
])):
logger.debug('spec looks like a shell job, processing now.')
spec = self.process_strings(spec, strings)
return self.generate_shell_job(spec)
elif 'tasks' in spec_keys:
logger.debug('spec looks like a shell job, processing now.')
spec = self.process_strings(spec, strings)
task_sequence = self.generate_sequence(spec, self.funcs)
return task_sequence.run, None
else:
if not self.funcs:
logger.warning('no functions available.')
logger.critical(
'spec does not match existing job type, returning early')
raise InvalidJob('invalid job type error')
@staticmethod
def get_dependency_list(spec):
"""
:param dict spec: A specification of a build task with a ``target``, to process.
:returns: A list that specifies all specified dependencies of that task.
"""
if 'dependency' in spec:
dependency = spec['dependency']
elif 'dep' in spec:
dependency = spec['dep']
elif 'deps' in spec:
dependency = spec['deps']
else:
logger.warning('no dependency in {0} spec'.format(spec['target']))
return list()
if isinstance(dependency, list):
return dependency
else:
return dependency.split()
def _process_dependency(self, spec):
"""
:param dict spec: The task specification imported from user input.
Wraps :meth:`~system.BuildSystemGenerator._process_stage()` and
modifies the internal strucutres associated with the
dependency tree.
"""
job = self._process_stage(spec)
dependencies = self.get_dependency_list(spec)
self.specs[spec['target']] = spec
if self.check.check(spec['target'], dependencies) is True:
msg = 'target {0} is older than dependency {1}: adding to build queue'
logger.info(msg.format(spec['target'], dependencies))
self._process_jobs[spec['target']] = (job, True)
else:
logger.info('rebuild not needed for {0}.')
self._process_jobs[spec['target']] = (job, False)
logger.debug('added {0} to dependency graph'.format(spec['target']))
self._process_tree[spec['target']] = self.get_dependency_list(spec)
def _process_job(self, spec, strings=None):
"""
:param dict spec: A dictionary of strings that describe a build job.
Processes the ``spec`` and attempts to create and add the resulting task
to the build system.
"""
if strings is not None:
for key in spec:
spec[key] = spec[key].format(strings)
if 'dependency' in spec or 'dep' in spec or 'deps' in spec:
if ('stage' in spec and 'target' in spec):
logger.error(
'{0} cannot have both a stage and target'.format(spec))
raise InvalidJob('stage cannot have "stage" and "target".')
if 'stage' in spec and not 'target' in spec:
spec['target'] = spec['stage']
del spec['stage']
if 'target' in spec:
self._process_dependency(spec)
logger.debug('added task to build {0}'.format(spec['target']))
return True
else:
if 'target' in spec and not 'stage' in spec:
spec['stage'] = spec['target']
del spec['target']
if 'stage' not in spec:
# put this into the BuildSystem stage to run after the dependency tree.
logger.warning(
'job lacks stage name, adding one to "{0}", please correct.'
.format(spec))
spec['stage'] = '__unspecified'
job = self._process_stage(spec)
if not self._stages.stage_exists(spec['stage']):
logger.debug('creating new stage named {0}'.format(
spec['stage']))
self._stages.new_stage(spec['stage'])
self._stages.stages[spec['stage']].add(job[0], job[1])
logger.debug('added job to stage: {0}'.format(spec['stage']))
return True
class BuildSystemGenerator(object):
"""
:class:`~system.BuildSystemGenerator` objects provide a unifying interface
for generating and running :class:`~system.BuildSystem` objects from
easy-to-edit sources. :class:`~system.BuildSystemGenerator` is the
fundamental basis of the :ref:`buildc` tool.
:param dict funcs: Optional. A dictionary that maps callable objects
(values), to identifiers used by build system definitions. If you do not
specify.
:class:`~system.BuildSystemGenerator` is designed to parse streams of YAML
documents that resemble the following:
.. code-block:: yaml
job: <func>
args: <<arg>,<list>>
stage: <str>
---
job: <func>
args: <<arg>,<list>>
target: <str>
dependency: <<str>,<list>>
---
dir: <<path>,<list>>
cmd: <str>
args: <<arg>,<list>>
stage: <str>
---
dir: <<path>,<list>>
cmd: <str>
args: <<arg>,<list>>
dependency: <<str>,<list>>
---
tasks:
- job: <func>
args: <<arg>,<list>>
- dir: <<path>,<list>>
cmd: <str>
args <<args>,<list>>
stage: <str>
...
See :doc:`/tutorial/use-buildc` for documentation of the input data
format and its use.
"""
def __init__(self, funcs=None):
if isinstance(funcs, dict):
self.funcs = funcs
logger.info('created build generator object with a default job mapping.')
else:
self.funcs = {}
logger.info('created build generator object with an empty job mapping.')
self._stages = BuildSystem()
logger.info('created empty build system object for build generator.')
self._process_jobs = {}
"Mapping of targets to job tuples."
self._process_tree = {}
"Internal representation of the dependency tree."
self.specs = {}
"Mapping of job specs targets to specs."
self.system = None
self._final = False
"""Internal switch that indicates a finalized method. See
:meth:`~system.BuildSystemGenerator.finlize()` for more information."""
self.check = DependencyChecks()
logger.info('created build system generator object')
@property
def check_method(self):
return self.check.check_method
@check_method.setter
def check_method(self, value):
if value not in self.check.checks:
pass
else:
self.check.check_method = value
def finalize(self):
"""
:raises: :exc:`~err.InvalidSystem` if you call
:meth:`~system.BuildSystemGenerator.finalize()` more than once on a
single :class:`~system.BuildSystemGenerator()` object.
You must call :meth:`~system.BuildSystemGenerator.finalize()` before
running the build system.
:meth:`~system.BuildSystemGenerator.finalize()` compiles the build
system. If there are no tasks with dependencies,
:attr:`~system.BuildSystemGenerator._stages` becomes the
:attr:`~system.BuildSystemGenerator.system` object. Otherwise,
:meth:`~system.BuildSystemGenerator.finalize()` orders the tasks with
dependencies, inserts them into a :class:`~stages.BuildSequence` object
before inserting the :attr:`~system.BuildSystemGenerator._stages` tasks.
"""
if self._final is False and self.system is None:
if len(self._process_tree) == 0:
logger.debug('no dependency tasks exist, trying to add build stages.')
if self._stages.count() > 0:
self.system = self._stages
logger.info('added tasks from build stages to build system.')
else:
logger.critical('cannot finalize empty generated BuildSystem object.')
raise InvalidSystem
elif len(self._process_tree) > 0:
self.system = BuildSystem(self._stages)
self._process = tsort(self._process_tree)
logger.debug('successfully sorted dependency tree.')
self._finalize_process_tree()
if self._stages.count() > 0:
self.system.extend(self._stages)
logger.info('added stages tasks to build system.')
self.system.close()
self._final = True
else:
logger.critical('cannot finalize object')
raise InvalidSystem
def _finalize_process_tree(self):
"""
Loops over the :attr:`~system.BuildSystemGenerator._process` list tree
created in the main :meth:`~system.BuildSystemGenerator.finalize()`
method, and adds tasks to :attr:`~system.BuildSystemGenerator.system`
object, which is itself a :class:`~system.BuildSystem` object.
While constructing the :attr:`~system.BuildSystemGenerator.system`
object, :meth:`~system.BuildSystemGenerator._finalize_process_tree()`
combines adjacent tasks that do not depend upon each other to increase
the potential for parallel execution.
:meth:`~system.BuildSystemGenerator._finalize_process`, calls
:meth:`~system.BuildSystemGenerator._add_tasks_to_stage()` which may
raise :exc:`~err.InvalidSystem` in the case of malformed tasks.
"""
total = len(self._process)
rebuilds_needed = False
idx = -1
stack = []
for i in self._process:
idx += 1
if rebuilds_needed is False:
if self._process_jobs[i][1] is False:
logger.debug('{0}: does not need a rebuild, passing'.format(i))
continue
elif self._process_jobs[i][1] is True:
logger.debug('{0}: needs rebuild.'.format(i))
rebuilds_needed = True
# rebuild needed here.
if idx+1 == total:
# last task in tree.
if stack:
# if there are non dependent items in the stack, we
# might as well add the current task there.
stack.append(i)
else:
# if there's no stack, we'll just add the task directly.
pass
else:
if i not in self._process_tree[self._process[idx]]:
stack.append(i)
logger.debug('adding task {0} to queue not continuing.'.format(i))
continue
else:
# here the previous stack doesn't depend on the
# current task. add current task to stack then build
# the stack in parallel.
stack.append(i)
logger.debug('adding task to the stage queue, but not continuing'.format(i))
self._add_tasks_to_stage(rebuilds_needed, idx, total, i, stack)
stack = []
def _add_tasks_to_stage(self, rebuilds_needed, idx, total, task, stack):
"""
:param bool rebuild_needed: ``True``, when the dependencies require a
rebuild of this target.
:param int idx: The index of the current task in the build dependency
chain.
:param int total: The total number of tasks in the dependency chain.
:param string task: The name of a task in the build process.
:param list stack: The list of non-dependent tasks that must be rebuilt,
not including the current task.
:raises: :exc:`~err.InvalidSystem` if its impossible to add a task in
the :class:`~system.BuildSystemGenerator` object to the finalized
build system object.
Called by :meth:`~system.BuildSystemGenerator._finalize_process_tree()`
to add and process tasks.
"""
if rebuilds_needed is True:
self.system.new_stage(task)
if stack:
for job in stack:
self.system.stages[task].add(self._process_jobs[job][0][0],
self._process_jobs[job][0][1])
else:
logger.debug('{0}: adding to rebuild queue'.format(task))
self.system.stages[task].add(self._process_jobs[task][0][0],
self._process_jobs[task][0][1])
elif rebuilds_needed is False:
logger.warning("dropping {0} task, no rebuild needed.".format(task))
return None
@staticmethod
def process_strings(spec, strings):
"""
:param string spec: A string, possibly with ``format()`` style tokens in curly braces.
:param dict strings: A mapping of strings to replacement values.
:raises: :exc:`python:TypeError` if ``strings`` is not a dict *and*
there may be a replacement string.
"""
if strings is None:
return spec
elif not isinstance(strings, dict):
logger.critical('replacement content must be a dictionary.')
raise TypeError
if isinstance(spec, dict):
out = {}
for k, v in spec.items():
if '{' in v and '}' in v:
try:
out[k] = v.format(**strings)
except KeyError as e:
msg = '{0} does not have {1} suitable replacement keys.'.format(strings, e)
logger.critical(msg)
raise InvalidJob(msg)
else:
out[k] = v
return out
else:
if '{' in spec and '}' in spec:
try:
spec = spec.format(**strings)
except KeyError as e:
msg = '{0} does not have {1} suitable replacement keys.'.format(spec, e)
logger.critical(msg)
raise InvalidJob(msg)
return spec
else:
return spec
@staticmethod
def generate_job(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names (i.e. ``spec.job`` to
functions.)
:returns: A tuple where the first item is a callable and the second item
is either a dict or a tuple of arguments, depending on the type of
the ``spec.args`` value.
:raises: :exc:`~err.InvalidStage` if ``spec.args`` is neither a
``dict``, ``list``, or ``tuple``.
"""
if isinstance(spec['args'], dict):
args = spec['args']
elif isinstance(spec['args'], list):
args = tuple(spec['args'])
elif isinstance(spec['args'], tuple):
args = spec['args']
else:
raise InvalidJob('args are malformed.')
if is_function(spec['job']):
action = spec['job']
else:
try:
action = funcs[spec['job']]
except KeyError:
msg = "{0} not in function specification with: {0} keys".format(spec['job'], funcs.keys())
logger.critical(msg)
raise InvalidJob(msg)
return action, args
@staticmethod
def generate_shell_job(spec):
"""
:param dict spec: A *job* specification.
:returns: A tuple where the first element is :mod:`python:subprocess`
:func:`~python:subprocess.call`, and the second item is a dict
``args``, a ``dir``, and ``cmd`` keys.
Takes a ``spec`` dict and returns a tuple to define a task.
"""
if isinstance(spec['cmd'], list):
cmd_str = spec['cmd']
else:
cmd_str = spec['cmd'].split()
if isinstance(spec['dir'], list):
base_path = os.path.sep.join(spec['dir'])
if spec['dir'][0].startswith(os.path.sep):
spec['dir'] = base_path
else:
spec['dir'] = os.path.abspath(base_path)
if isinstance(spec['args'], list):
cmd_str.extend(spec['args'])
else:
cmd_str.extend(spec['args'].split())
return subprocess.call, dict(cwd=spec['dir'],
args=cmd_str)
@staticmethod
def generate_sequence(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names to callable objects.
:returns: A :class:`~stages.BuildSequence()` object.
:raises: An exception when ``spec`` does not have a ``task`` element or
when ``task.spec`` is not a list.
Process a job spec that describes a sequence of tasks and returns a
:class:`~stages.BuildSequence()` object for inclusion in a
:class:`~system.BuildSystem()` object.
"""
sequence = BuildSequence()
if 'tasks' not in spec or not isinstance(spec['tasks'], list):
raise InvalidStage
else:
for task in spec['tasks']:
if 'job' in task:
job, args = BuildSystemGenerator.generate_job(task, funcs)
elif 'cmd' in task:
job, args = BuildSystemGenerator.generate_shell_job(task)
sequence.add(job, args)
sequence.close()
return sequence
def add_task(self, name, func):
"""
:param string name: The identifier of a callable.
:param callable func: A callable object.
:raises: :exc:`~err.InvaidJob` if ``func`` is not callable.
Adds a callable object to the :attr:`~system.BuildSystemGenerator.funcs`
attribute with the identifier ``name``.
"""
if not is_function(func):
logger.critical('cannot add tasks that are not callables ({0}).'.format(name))
raise InvalidJob("{0} is not callable.".format(func))
logger.debug('adding task named {0}'.format(name))
self.funcs[name] = func
def ingest(self, jobs, strings=None):
"""
:param iterable jobs: An interable object that contains build job
specifications.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
:returns: The number of jobs added to the build system.
For every document
"""
job_count = 0
for spec in jobs:
self._process_job(spec, strings)
job_count += 1
logger.debug('loaded {0} jobs'.format(job_count))
return job_count
def ingest_yaml(self, filename, strings=None):
"""
Wraps :meth:`~BuildSystemGenerator.ingest()`.
:param string filename: The fully qualified path name of a :term:`YAML`
file that contains a build system specification.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
For every document in the YAML file specified by ``filename``, calls
:meth:`~system.BuildSystemGenerator._process_job()` to add that job to
the :attr:`~system.BuildSystemGenerator.system` object.
"""
logger.debug('opening yaml file {0}'.format(filename))
try:
with open(filename, 'r') as f:
try:
jobs = yaml.safe_load_all(f)
except NameError:
msg = 'attempting to load a yaml definition without PyYAML installed.'
logger.critical(msg)
raise StageRunError(msg)
job_count = self.ingest(jobs, strings)
logger.debug('loaded {0} jobs from {1}'.format(job_count, filename))
except IOError:
logger.warning('file {0} does not exist'.format(filename))
def ingest_json(self, filename, strings=None):
"""
Wraps :meth:`~BuildSystemGenerator.ingest()`.
:param string filename: The fully qualified path name of a :term:`JSON`
file that contains a build system specification.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
For every document in the JSON file specified by ``filename``, calls
:meth:`~system.BuildSystemGenerator._process_job()` to add that job to
the :attr:`~system.BuildSystemGenerator.system` object.
"""
logger.debug('opening json file {0}'.format(filename))
try:
with open(filename, 'r') as f:
jobs = json.load(f)
job_count = self.ingest(jobs, strings)
logger.debug('loaded {0} jobs from {1}'.format(job_count, filename))
except IOError:
logger.warning('file {0} does not exist'.format(filename))
def _process_stage(self, spec, spec_keys=None, strings=None):
"""
:param dict spec: The task specification imported from user input.
:param set spec_keys: A set containing the top level keys in the set.
:param dict strings: Optional. A dictionary of strings mapping to
strings to use as replacement keys for spec content.
:returns: A two item tuple, containing a function and a list of arguments.
:raises: :exc:`~err.InvalidJob` if the schema of ``spec`` is not recognized.
Based on the schema of the ``spec``, creates tasks and sequences to add
to a BuildSystem using, as appropriate one of the following methods:
- :meth:`~system.BuildSystemGenerator.generate_job()`,
- :meth:`~system.BuildSystemGenerator.generate_shell_job()`, or
- :meth:`~system.BuildSystemGenerator.generate_sequence()`.
"""
if spec_keys is None:
spec_keys = set(spec.keys())
elif isinstance(spec_keys, set):
logger.debug('using {0} for spec_keys as an optimization'.format(spec_keys))
else:
logger.error('spec_keys argument {0} must be a set'.format(spec_keys))
raise InvalidJob('problem with spec_keys')
if spec_keys.issuperset(set(['job', 'args'])):
logger.debug('spec looks like a pure python job, processing now.')
spec = self.process_strings(spec, strings)
return self.generate_job(spec, self.funcs)
elif spec_keys.issuperset(set(['dir', 'cmd', 'args', ])):
logger.debug('spec looks like a shell job, processing now.')
spec = self.process_strings(spec, strings)
return self.generate_shell_job(spec)
elif 'tasks' in spec_keys:
logger.debug('spec looks like a shell job, processing now.')
spec = self.process_strings(spec, strings)
task_sequence = self.generate_sequence(spec, self.funcs)
return task_sequence.run, None
else:
if not self.funcs:
logger.warning('no functions available.')
logger.critical('spec does not match existing job type, returning early')
raise InvalidJob('invalid job type error')
@staticmethod
def get_dependency_list(spec):
"""
:param dict spec: A specification of a build task with a ``target``, to process.
:returns: A list that specifies all specified dependencies of that task.
"""
if 'dependency' in spec:
dependency = spec['dependency']
elif 'dep' in spec:
dependency = spec['dep']
elif 'deps' in spec:
dependency = spec['deps']
else:
logger.warning('no dependency in {0} spec'.format(spec['target']))
return list()
if isinstance(dependency, list):
return dependency
else:
return dependency.split()
def _process_dependency(self, spec):
"""
:param dict spec: The task specification imported from user input.
Wraps :meth:`~system.BuildSystemGenerator._process_stage()` and
modifies the internal strucutres associated with the
dependency tree.
"""
job = self._process_stage(spec)
dependencies = self.get_dependency_list(spec)
self.specs[spec['target']] = spec
if self.check.check(spec['target'], dependencies) is True:
msg = 'target {0} is older than dependency {1}: adding to build queue'
logger.info(msg.format(spec['target'], dependencies))
self._process_jobs[spec['target']] = (job, True)
else:
logger.info('rebuild not needed for {0}.')
self._process_jobs[spec['target']] = (job, False)
logger.debug('added {0} to dependency graph'.format(spec['target']))
self._process_tree[spec['target']] = self.get_dependency_list(spec)
def _process_job(self, spec, strings=None):
"""
:param dict spec: A dictionary of strings that describe a build job.
Processes the ``spec`` and attempts to create and add the resulting task
to the build system.
"""
if strings is not None:
for key in spec:
spec[key] = spec[key].format(strings)
if 'dependency' in spec or 'dep' in spec or 'deps' in spec:
if ('stage' in spec and 'target' in spec):
logger.error('{0} cannot have both a stage and target'.format(spec))
raise InvalidJob('stage cannot have "stage" and "target".')
if 'stage' in spec and not 'target' in spec:
spec['target'] = spec['stage']
del spec['stage']
if 'target' in spec:
self._process_dependency(spec)
logger.debug('added task to build {0}'.format(spec['target']))
return True
else:
if 'target' in spec and not 'stage' in spec:
spec['stage'] = spec['target']
del spec['target']
if 'stage' not in spec:
# put this into the BuildSystem stage to run after the dependency tree.
logger.warning('job lacks stage name, adding one to "{0}", please correct.'.format(spec))
spec['stage'] = '__unspecified'
job = self._process_stage(spec)
if not self._stages.stage_exists(spec['stage']):
logger.debug('creating new stage named {0}'.format(spec['stage']))
self._stages.new_stage(spec['stage'])
self._stages.stages[spec['stage']].add(job[0], job[1])
logger.debug('added job to stage: {0}'.format(spec['stage']))
return True
def setUp(self):
self.d = DependencyChecks()
self.fn_a = 'fn_a'
self.fn_b = 'fn_b'
class TestDependencyChecking(TestCase):
@classmethod
def setUp(self):
self.d = DependencyChecks()
self.fn_a = 'fn_a'
self.fn_b = 'fn_b'
@classmethod
def tearDown(self):
for fn in [self.fn_a, self.fn_b]:
if os.path.exists(fn):
os.remove(fn)
def ensure_clean(self):
self.assertFalse(os.path.exists(self.fn_a))
self.assertFalse(os.path.exists(self.fn_b))
def test_basic(self):
self.ensure_clean()
touch(self.fn_a)
self.assertTrue(os.path.exists(self.fn_a))
self.assertFalse(os.path.exists(self.fn_b))
def test_basic_alt(self):
self.ensure_clean()
touch(self.fn_b)
self.assertTrue(os.path.exists(self.fn_b))
self.assertFalse(os.path.exists(self.fn_a))
def test_default_method(self):
self.ensure_clean()
self.assertTrue(self.d.check_method, 'mtime')
def test_setting_valid_methods(self):
self.ensure_clean()
for method in ['force', 'ignore', 'hash', 'mtime']:
self.d.check_method = method
self.assertTrue(self.d.check_method, method)
self.d.check_method = 'mtime'
self.assertTrue(self.d.check_method, 'mtime')
def test_setting_invalid_method(self):
self.ensure_clean()
with self.assertRaises(DependencyCheckError):
self.d.check_method = 'magic'
def test_mtime_rebuild(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.assertTrue(self.d.check_method, 'mtime')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_mtime_no_rebuild(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.assertTrue(self.d.check_method, 'mtime')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_mtime_rebuild_no_target(self):
self.ensure_clean()
touch(self.fn_b)
self.assertTrue(self.d.check_method, 'mtime')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild(self):
self.ensure_clean()
write(self.fn_a, 'aaa')
write(self.fn_b, 'bbb')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild_ignoring_update_order(self):
self.ensure_clean()
write(self.fn_b, 'bbb')
breath()
breath()
write(self.fn_a, 'aaa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_hash_no_rebuild(self):
self.ensure_clean()
write(self.fn_b, 'aaa')
write(self.fn_a, 'aaa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_hash_rebuild_no_target(self):
self.ensure_clean()
write(self.fn_b, 'aa')
self.d.check_method = 'hash'
self.assertTrue(self.d.check_method, 'hash')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_non_existing(self):
self.ensure_clean()
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_with_files(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_force_with_reversed_files(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.d.check_method = 'force'
self.assertTrue(self.d.check_method, 'force')
self.assertTrue(self.d.check(self.fn_a, self.fn_b))
def test_ignore_non_existing(self):
self.ensure_clean()
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_ignore_with_files(self):
self.ensure_clean()
touch(self.fn_a)
breath()
touch(self.fn_b)
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
def test_ignore_with_reversed_files(self):
self.ensure_clean()
touch(self.fn_b)
breath()
touch(self.fn_a)
self.d.check_method = 'ignore'
self.assertTrue(self.d.check_method, 'ignore')
self.assertFalse(self.d.check(self.fn_a, self.fn_b))
