def __init__(self, funcs=None): if isinstance(funcs, dict): self.funcs = funcs logger.info( 'created build generator object with a default job mapping.') else: self.funcs = {} logger.info( 'created build generator object with an empty job mapping.') self._stages = BuildSystem() logger.info('created empty build system object for build generator.') self._process_jobs = {} "Mapping of targets to job tuples." self._process_tree = {} "Internal representation of the dependency tree." self.specs = {} "Mapping of job specs targets to specs." self.system = None self._final = False """Internal switch that indicates a finalized method. See :meth:`~system.BuildSystemGenerator.finlize()` for more information.""" self.check = DependencyChecks() logger.info('created build system generator object')
def __init__(self, funcs=None): if isinstance(funcs, dict): self.funcs = funcs logger.info('created build generator object with a default job mapping.') else: self.funcs = {} logger.info('created build generator object with an empty job mapping.') self._stages = BuildSystem() logger.info('created empty build system object for build generator.') self._process_jobs = {} "Mapping of targets to job tuples." self._process_tree = {} "Internal representation of the dependency tree." self.specs = {} "Mapping of job specs targets to specs." self.system = None self._final = False """Internal switch that indicates a finalized method. See :meth:`~system.BuildSystemGenerator.finlize()` for more information.""" self.check = DependencyChecks() logger.info('created build system generator object')
def setUp(self): self.d = DependencyChecks() self.fn_a = 'fn_a' self.fn_b = 'fn_b'
class TestDependencyChecking(TestCase): @classmethod def setUp(self): self.d = DependencyChecks() self.fn_a = 'fn_a' self.fn_b = 'fn_b' @classmethod def tearDown(self): for fn in [ self.fn_a, self.fn_b ]: if os.path.exists(fn): os.remove(fn) def ensure_clean(self): self.assertFalse(os.path.exists(self.fn_a)) self.assertFalse(os.path.exists(self.fn_b)) def test_basic(self): self.ensure_clean() touch(self.fn_a) self.assertTrue(os.path.exists(self.fn_a)) self.assertFalse(os.path.exists(self.fn_b)) def test_basic_alt(self): self.ensure_clean() touch(self.fn_b) self.assertTrue(os.path.exists(self.fn_b)) self.assertFalse(os.path.exists(self.fn_a)) def test_default_method(self): self.ensure_clean() self.assertTrue(self.d.check_method, 'mtime') def test_setting_valid_methods(self): self.ensure_clean() for method in ['force', 'ignore', 'hash', 'mtime']: self.d.check_method = method self.assertTrue(self.d.check_method, method) self.d.check_method = 'mtime' self.assertTrue(self.d.check_method, 'mtime') def test_setting_invalid_method(self): self.ensure_clean() with self.assertRaises(DependencyCheckError): self.d.check_method = 'magic' def test_mtime_rebuild(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.assertTrue(self.d.check_method, 'mtime') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_mtime_no_rebuild(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.assertTrue(self.d.check_method, 'mtime') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_mtime_rebuild_no_target(self): self.ensure_clean() touch(self.fn_b) self.assertTrue(self.d.check_method, 'mtime') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild(self): self.ensure_clean() write(self.fn_a, 'aaa') write(self.fn_b, 'bbb') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild_ignoring_update_order(self): self.ensure_clean() write(self.fn_b, 'bbb') breath() breath() write(self.fn_a, 'aaa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_no_rebuild(self): self.ensure_clean() write(self.fn_b, 'aaa') write(self.fn_a, 'aaa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild_no_target(self): self.ensure_clean() write(self.fn_b, 'aa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_non_existing(self): self.ensure_clean() self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_with_files(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_with_reversed_files(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_ignore_non_existing(self): self.ensure_clean() self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_ignore_with_files(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_ignore_with_reversed_files(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b))
class BuildSystemGenerator(object): """ :class:`~system.BuildSystemGenerator` objects provide a unifying interface for generating and running :class:`~system.BuildSystem` objects from easy-to-edit sources. :class:`~system.BuildSystemGenerator` is the fundamental basis of the :ref:`buildc` tool. :param dict funcs: Optional. A dictionary that maps callable objects (values), to identifiers used by build system definitions. If you do not specify. :class:`~system.BuildSystemGenerator` is designed to parse streams of YAML documents that resemble the following: .. code-block:: yaml job: <func> args: <<arg>,<list>> stage: <str> --- job: <func> args: <<arg>,<list>> target: <str> dependency: <<str>,<list>> --- dir: <<path>,<list>> cmd: <str> args: <<arg>,<list>> stage: <str> --- dir: <<path>,<list>> cmd: <str> args: <<arg>,<list>> dependency: <<str>,<list>> --- tasks: - job: <func> args: <<arg>,<list>> - dir: <<path>,<list>> cmd: <str> args <<args>,<list>> stage: <str> ... See :doc:`/tutorial/use-buildc` for documentation of the input data format and its use. """ def __init__(self, funcs=None): if isinstance(funcs, dict): self.funcs = funcs logger.info( 'created build generator object with a default job mapping.') else: self.funcs = {} logger.info( 'created build generator object with an empty job mapping.') self._stages = BuildSystem() logger.info('created empty build system object for build generator.') self._process_jobs = {} "Mapping of targets to job tuples." self._process_tree = {} "Internal representation of the dependency tree." self.specs = {} "Mapping of job specs targets to specs." self.system = None self._final = False """Internal switch that indicates a finalized method. See :meth:`~system.BuildSystemGenerator.finlize()` for more information.""" self.check = DependencyChecks() logger.info('created build system generator object') @property def check_method(self): return self.check.check_method @check_method.setter def check_method(self, value): if value not in self.check.checks: pass else: self.check.check_method = value def finalize(self): """ :raises: :exc:`~err.InvalidSystem` if you call :meth:`~system.BuildSystemGenerator.finalize()` more than once on a single :class:`~system.BuildSystemGenerator()` object. You must call :meth:`~system.BuildSystemGenerator.finalize()` before running the build system. :meth:`~system.BuildSystemGenerator.finalize()` compiles the build system. If there are no tasks with dependencies, :attr:`~system.BuildSystemGenerator._stages` becomes the :attr:`~system.BuildSystemGenerator.system` object. Otherwise, :meth:`~system.BuildSystemGenerator.finalize()` orders the tasks with dependencies, inserts them into a :class:`~stages.BuildSequence` object before inserting the :attr:`~system.BuildSystemGenerator._stages` tasks. """ if self._final is False and self.system is None: if len(self._process_tree) == 0: logger.debug( 'no dependency tasks exist, trying to add build stages.') if self._stages.count() > 0: self.system = self._stages logger.info( 'added tasks from build stages to build system.') else: logger.critical( 'cannot finalize empty generated BuildSystem object.') raise InvalidSystem elif len(self._process_tree) > 0: self.system = BuildSystem(self._stages) self._process = tsort(self._process_tree) logger.debug('successfully sorted dependency tree.') self._finalize_process_tree() if self._stages.count() > 0: self.system.extend(self._stages) logger.info('added stages tasks to build system.') self.system.close() self._final = True else: logger.critical('cannot finalize object') raise InvalidSystem def _finalize_process_tree(self): """ Loops over the :attr:`~system.BuildSystemGenerator._process` list tree created in the main :meth:`~system.BuildSystemGenerator.finalize()` method, and adds tasks to :attr:`~system.BuildSystemGenerator.system` object, which is itself a :class:`~system.BuildSystem` object. While constructing the :attr:`~system.BuildSystemGenerator.system` object, :meth:`~system.BuildSystemGenerator._finalize_process_tree()` combines adjacent tasks that do not depend upon each other to increase the potential for parallel execution. :meth:`~system.BuildSystemGenerator._finalize_process`, calls :meth:`~system.BuildSystemGenerator._add_tasks_to_stage()` which may raise :exc:`~err.InvalidSystem` in the case of malformed tasks. """ total = len(self._process) rebuilds_needed = False idx = -1 stack = [] for i in self._process: idx += 1 if rebuilds_needed is False: if self._process_jobs[i][1] is False: logger.debug( '{0}: does not need a rebuild, passing'.format(i)) continue elif self._process_jobs[i][1] is True: logger.debug('{0}: needs rebuild.'.format(i)) rebuilds_needed = True # rebuild needed here. if idx + 1 == total: # last task in tree. if stack: # if there are non dependent items in the stack, we # might as well add the current task there. stack.append(i) else: # if there's no stack, we'll just add the task directly. pass else: if i not in self._process_tree[self._process[idx]]: stack.append(i) logger.debug( 'adding task {0} to queue not continuing.'.format(i)) continue else: # here the previous stack doesn't depend on the # current task. add current task to stack then build # the stack in parallel. stack.append(i) logger.debug( 'adding task to the stage queue, but not continuing'. format(i)) self._add_tasks_to_stage(rebuilds_needed, idx, total, i, stack) stack = [] def _add_tasks_to_stage(self, rebuilds_needed, idx, total, task, stack): """ :param bool rebuild_needed: ``True``, when the dependencies require a rebuild of this target. :param int idx: The index of the current task in the build dependency chain. :param int total: The total number of tasks in the dependency chain. :param string task: The name of a task in the build process. :param list stack: The list of non-dependent tasks that must be rebuilt, not including the current task. :raises: :exc:`~err.InvalidSystem` if its impossible to add a task in the :class:`~system.BuildSystemGenerator` object to the finalized build system object. Called by :meth:`~system.BuildSystemGenerator._finalize_process_tree()` to add and process tasks. """ if rebuilds_needed is True: self.system.new_stage(task) if stack: for job in stack: self.system.stages[task].add(self._process_jobs[job][0][0], self._process_jobs[job][0][1]) else: logger.debug('{0}: adding to rebuild queue'.format(task)) self.system.stages[task].add(self._process_jobs[task][0][0], self._process_jobs[task][0][1]) elif rebuilds_needed is False: logger.warning( "dropping {0} task, no rebuild needed.".format(task)) return None @staticmethod def process_strings(spec, strings): """ :param string spec: A string, possibly with ``format()`` style tokens in curly braces. :param dict strings: A mapping of strings to replacement values. :raises: :exc:`python:TypeError` if ``strings`` is not a dict *and* there may be a replacement string. """ if strings is None: return spec elif not isinstance(strings, dict): logger.critical('replacement content must be a dictionary.') raise TypeError if isinstance(spec, dict): out = {} for k, v in spec.items(): if '{' in v and '}' in v: try: out[k] = v.format(**strings) except KeyError as e: msg = '{0} does not have {1} suitable replacement keys.'.format( strings, e) logger.critical(msg) raise InvalidJob(msg) else: out[k] = v return out else: if '{' in spec and '}' in spec: try: spec = spec.format(**strings) except KeyError as e: msg = '{0} does not have {1} suitable replacement keys.'.format( spec, e) logger.critical(msg) raise InvalidJob(msg) return spec else: return spec @staticmethod def generate_job(spec, funcs): """ :param dict spec: A *job* specification. :param dict funcs: A dictionary mapping names (i.e. ``spec.job`` to functions.) :returns: A tuple where the first item is a callable and the second item is either a dict or a tuple of arguments, depending on the type of the ``spec.args`` value. :raises: :exc:`~err.InvalidStage` if ``spec.args`` is neither a ``dict``, ``list``, or ``tuple``. """ if isinstance(spec['args'], dict): args = spec['args'] elif isinstance(spec['args'], list): args = tuple(spec['args']) elif isinstance(spec['args'], tuple): args = spec['args'] else: raise InvalidJob('args are malformed.') if is_function(spec['job']): action = spec['job'] else: try: action = funcs[spec['job']] except KeyError: msg = "{0} not in function specification with: {0} keys".format( spec['job'], funcs.keys()) logger.critical(msg) raise InvalidJob(msg) return action, args @staticmethod def generate_shell_job(spec): """ :param dict spec: A *job* specification. :returns: A tuple where the first element is :mod:`python:subprocess` :func:`~python:subprocess.call`, and the second item is a dict ``args``, a ``dir``, and ``cmd`` keys. Takes a ``spec`` dict and returns a tuple to define a task. """ if isinstance(spec['cmd'], list): cmd_str = spec['cmd'] else: cmd_str = spec['cmd'].split() if isinstance(spec['dir'], list): base_path = os.path.sep.join(spec['dir']) if spec['dir'][0].startswith(os.path.sep): spec['dir'] = base_path else: spec['dir'] = os.path.abspath(base_path) if isinstance(spec['args'], list): cmd_str.extend(spec['args']) else: cmd_str.extend(spec['args'].split()) return subprocess.call, dict(cwd=spec['dir'], args=cmd_str) @staticmethod def generate_sequence(spec, funcs): """ :param dict spec: A *job* specification. :param dict funcs: A dictionary mapping names to callable objects. :returns: A :class:`~stages.BuildSequence()` object. :raises: An exception when ``spec`` does not have a ``task`` element or when ``task.spec`` is not a list. Process a job spec that describes a sequence of tasks and returns a :class:`~stages.BuildSequence()` object for inclusion in a :class:`~system.BuildSystem()` object. """ sequence = BuildSequence() if 'tasks' not in spec or not isinstance(spec['tasks'], list): raise InvalidStage else: for task in spec['tasks']: if 'job' in task: job, args = BuildSystemGenerator.generate_job(task, funcs) elif 'cmd' in task: job, args = BuildSystemGenerator.generate_shell_job(task) sequence.add(job, args) sequence.close() return sequence def add_task(self, name, func): """ :param string name: The identifier of a callable. :param callable func: A callable object. :raises: :exc:`~err.InvaidJob` if ``func`` is not callable. Adds a callable object to the :attr:`~system.BuildSystemGenerator.funcs` attribute with the identifier ``name``. """ if not is_function(func): logger.critical( 'cannot add tasks that are not callables ({0}).'.format(name)) raise InvalidJob("{0} is not callable.".format(func)) logger.debug('adding task named {0}'.format(name)) self.funcs[name] = func def ingest(self, jobs, strings=None): """ :param iterable jobs: An interable object that contains build job specifications. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. :returns: The number of jobs added to the build system. For every document """ job_count = 0 for spec in jobs: self._process_job(spec, strings) job_count += 1 logger.debug('loaded {0} jobs'.format(job_count)) return job_count def ingest_yaml(self, filename, strings=None): """ Wraps :meth:`~BuildSystemGenerator.ingest()`. :param string filename: The fully qualified path name of a :term:`YAML` file that contains a build system specification. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. For every document in the YAML file specified by ``filename``, calls :meth:`~system.BuildSystemGenerator._process_job()` to add that job to the :attr:`~system.BuildSystemGenerator.system` object. """ logger.debug('opening yaml file {0}'.format(filename)) try: with open(filename, 'r') as f: try: jobs = yaml.safe_load_all(f) except NameError: msg = 'attempting to load a yaml definition without PyYAML installed.' logger.critical(msg) raise StageRunError(msg) job_count = self.ingest(jobs, strings) logger.debug('loaded {0} jobs from {1}'.format( job_count, filename)) except IOError: logger.warning('file {0} does not exist'.format(filename)) def ingest_json(self, filename, strings=None): """ Wraps :meth:`~BuildSystemGenerator.ingest()`. :param string filename: The fully qualified path name of a :term:`JSON` file that contains a build system specification. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. For every document in the JSON file specified by ``filename``, calls :meth:`~system.BuildSystemGenerator._process_job()` to add that job to the :attr:`~system.BuildSystemGenerator.system` object. """ logger.debug('opening json file {0}'.format(filename)) try: with open(filename, 'r') as f: jobs = json.load(f) job_count = self.ingest(jobs, strings) logger.debug('loaded {0} jobs from {1}'.format( job_count, filename)) except IOError: logger.warning('file {0} does not exist'.format(filename)) def _process_stage(self, spec, spec_keys=None, strings=None): """ :param dict spec: The task specification imported from user input. :param set spec_keys: A set containing the top level keys in the set. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. :returns: A two item tuple, containing a function and a list of arguments. :raises: :exc:`~err.InvalidJob` if the schema of ``spec`` is not recognized. Based on the schema of the ``spec``, creates tasks and sequences to add to a BuildSystem using, as appropriate one of the following methods: - :meth:`~system.BuildSystemGenerator.generate_job()`, - :meth:`~system.BuildSystemGenerator.generate_shell_job()`, or - :meth:`~system.BuildSystemGenerator.generate_sequence()`. """ if spec_keys is None: spec_keys = set(spec.keys()) elif isinstance(spec_keys, set): logger.debug( 'using {0} for spec_keys as an optimization'.format(spec_keys)) else: logger.error( 'spec_keys argument {0} must be a set'.format(spec_keys)) raise InvalidJob('problem with spec_keys') if spec_keys.issuperset(set(['job', 'args'])): logger.debug('spec looks like a pure python job, processing now.') spec = self.process_strings(spec, strings) return self.generate_job(spec, self.funcs) elif spec_keys.issuperset(set([ 'dir', 'cmd', 'args', ])): logger.debug('spec looks like a shell job, processing now.') spec = self.process_strings(spec, strings) return self.generate_shell_job(spec) elif 'tasks' in spec_keys: logger.debug('spec looks like a shell job, processing now.') spec = self.process_strings(spec, strings) task_sequence = self.generate_sequence(spec, self.funcs) return task_sequence.run, None else: if not self.funcs: logger.warning('no functions available.') logger.critical( 'spec does not match existing job type, returning early') raise InvalidJob('invalid job type error') @staticmethod def get_dependency_list(spec): """ :param dict spec: A specification of a build task with a ``target``, to process. :returns: A list that specifies all specified dependencies of that task. """ if 'dependency' in spec: dependency = spec['dependency'] elif 'dep' in spec: dependency = spec['dep'] elif 'deps' in spec: dependency = spec['deps'] else: logger.warning('no dependency in {0} spec'.format(spec['target'])) return list() if isinstance(dependency, list): return dependency else: return dependency.split() def _process_dependency(self, spec): """ :param dict spec: The task specification imported from user input. Wraps :meth:`~system.BuildSystemGenerator._process_stage()` and modifies the internal strucutres associated with the dependency tree. """ job = self._process_stage(spec) dependencies = self.get_dependency_list(spec) self.specs[spec['target']] = spec if self.check.check(spec['target'], dependencies) is True: msg = 'target {0} is older than dependency {1}: adding to build queue' logger.info(msg.format(spec['target'], dependencies)) self._process_jobs[spec['target']] = (job, True) else: logger.info('rebuild not needed for {0}.') self._process_jobs[spec['target']] = (job, False) logger.debug('added {0} to dependency graph'.format(spec['target'])) self._process_tree[spec['target']] = self.get_dependency_list(spec) def _process_job(self, spec, strings=None): """ :param dict spec: A dictionary of strings that describe a build job. Processes the ``spec`` and attempts to create and add the resulting task to the build system. """ if strings is not None: for key in spec: spec[key] = spec[key].format(strings) if 'dependency' in spec or 'dep' in spec or 'deps' in spec: if ('stage' in spec and 'target' in spec): logger.error( '{0} cannot have both a stage and target'.format(spec)) raise InvalidJob('stage cannot have "stage" and "target".') if 'stage' in spec and not 'target' in spec: spec['target'] = spec['stage'] del spec['stage'] if 'target' in spec: self._process_dependency(spec) logger.debug('added task to build {0}'.format(spec['target'])) return True else: if 'target' in spec and not 'stage' in spec: spec['stage'] = spec['target'] del spec['target'] if 'stage' not in spec: # put this into the BuildSystem stage to run after the dependency tree. logger.warning( 'job lacks stage name, adding one to "{0}", please correct.' .format(spec)) spec['stage'] = '__unspecified' job = self._process_stage(spec) if not self._stages.stage_exists(spec['stage']): logger.debug('creating new stage named {0}'.format( spec['stage'])) self._stages.new_stage(spec['stage']) self._stages.stages[spec['stage']].add(job[0], job[1]) logger.debug('added job to stage: {0}'.format(spec['stage'])) return True
class BuildSystemGenerator(object): """ :class:`~system.BuildSystemGenerator` objects provide a unifying interface for generating and running :class:`~system.BuildSystem` objects from easy-to-edit sources. :class:`~system.BuildSystemGenerator` is the fundamental basis of the :ref:`buildc` tool. :param dict funcs: Optional. A dictionary that maps callable objects (values), to identifiers used by build system definitions. If you do not specify. :class:`~system.BuildSystemGenerator` is designed to parse streams of YAML documents that resemble the following: .. code-block:: yaml job: <func> args: <<arg>,<list>> stage: <str> --- job: <func> args: <<arg>,<list>> target: <str> dependency: <<str>,<list>> --- dir: <<path>,<list>> cmd: <str> args: <<arg>,<list>> stage: <str> --- dir: <<path>,<list>> cmd: <str> args: <<arg>,<list>> dependency: <<str>,<list>> --- tasks: - job: <func> args: <<arg>,<list>> - dir: <<path>,<list>> cmd: <str> args <<args>,<list>> stage: <str> ... See :doc:`/tutorial/use-buildc` for documentation of the input data format and its use. """ def __init__(self, funcs=None): if isinstance(funcs, dict): self.funcs = funcs logger.info('created build generator object with a default job mapping.') else: self.funcs = {} logger.info('created build generator object with an empty job mapping.') self._stages = BuildSystem() logger.info('created empty build system object for build generator.') self._process_jobs = {} "Mapping of targets to job tuples." self._process_tree = {} "Internal representation of the dependency tree." self.specs = {} "Mapping of job specs targets to specs." self.system = None self._final = False """Internal switch that indicates a finalized method. See :meth:`~system.BuildSystemGenerator.finlize()` for more information.""" self.check = DependencyChecks() logger.info('created build system generator object') @property def check_method(self): return self.check.check_method @check_method.setter def check_method(self, value): if value not in self.check.checks: pass else: self.check.check_method = value def finalize(self): """ :raises: :exc:`~err.InvalidSystem` if you call :meth:`~system.BuildSystemGenerator.finalize()` more than once on a single :class:`~system.BuildSystemGenerator()` object. You must call :meth:`~system.BuildSystemGenerator.finalize()` before running the build system. :meth:`~system.BuildSystemGenerator.finalize()` compiles the build system. If there are no tasks with dependencies, :attr:`~system.BuildSystemGenerator._stages` becomes the :attr:`~system.BuildSystemGenerator.system` object. Otherwise, :meth:`~system.BuildSystemGenerator.finalize()` orders the tasks with dependencies, inserts them into a :class:`~stages.BuildSequence` object before inserting the :attr:`~system.BuildSystemGenerator._stages` tasks. """ if self._final is False and self.system is None: if len(self._process_tree) == 0: logger.debug('no dependency tasks exist, trying to add build stages.') if self._stages.count() > 0: self.system = self._stages logger.info('added tasks from build stages to build system.') else: logger.critical('cannot finalize empty generated BuildSystem object.') raise InvalidSystem elif len(self._process_tree) > 0: self.system = BuildSystem(self._stages) self._process = tsort(self._process_tree) logger.debug('successfully sorted dependency tree.') self._finalize_process_tree() if self._stages.count() > 0: self.system.extend(self._stages) logger.info('added stages tasks to build system.') self.system.close() self._final = True else: logger.critical('cannot finalize object') raise InvalidSystem def _finalize_process_tree(self): """ Loops over the :attr:`~system.BuildSystemGenerator._process` list tree created in the main :meth:`~system.BuildSystemGenerator.finalize()` method, and adds tasks to :attr:`~system.BuildSystemGenerator.system` object, which is itself a :class:`~system.BuildSystem` object. While constructing the :attr:`~system.BuildSystemGenerator.system` object, :meth:`~system.BuildSystemGenerator._finalize_process_tree()` combines adjacent tasks that do not depend upon each other to increase the potential for parallel execution. :meth:`~system.BuildSystemGenerator._finalize_process`, calls :meth:`~system.BuildSystemGenerator._add_tasks_to_stage()` which may raise :exc:`~err.InvalidSystem` in the case of malformed tasks. """ total = len(self._process) rebuilds_needed = False idx = -1 stack = [] for i in self._process: idx += 1 if rebuilds_needed is False: if self._process_jobs[i][1] is False: logger.debug('{0}: does not need a rebuild, passing'.format(i)) continue elif self._process_jobs[i][1] is True: logger.debug('{0}: needs rebuild.'.format(i)) rebuilds_needed = True # rebuild needed here. if idx+1 == total: # last task in tree. if stack: # if there are non dependent items in the stack, we # might as well add the current task there. stack.append(i) else: # if there's no stack, we'll just add the task directly. pass else: if i not in self._process_tree[self._process[idx]]: stack.append(i) logger.debug('adding task {0} to queue not continuing.'.format(i)) continue else: # here the previous stack doesn't depend on the # current task. add current task to stack then build # the stack in parallel. stack.append(i) logger.debug('adding task to the stage queue, but not continuing'.format(i)) self._add_tasks_to_stage(rebuilds_needed, idx, total, i, stack) stack = [] def _add_tasks_to_stage(self, rebuilds_needed, idx, total, task, stack): """ :param bool rebuild_needed: ``True``, when the dependencies require a rebuild of this target. :param int idx: The index of the current task in the build dependency chain. :param int total: The total number of tasks in the dependency chain. :param string task: The name of a task in the build process. :param list stack: The list of non-dependent tasks that must be rebuilt, not including the current task. :raises: :exc:`~err.InvalidSystem` if its impossible to add a task in the :class:`~system.BuildSystemGenerator` object to the finalized build system object. Called by :meth:`~system.BuildSystemGenerator._finalize_process_tree()` to add and process tasks. """ if rebuilds_needed is True: self.system.new_stage(task) if stack: for job in stack: self.system.stages[task].add(self._process_jobs[job][0][0], self._process_jobs[job][0][1]) else: logger.debug('{0}: adding to rebuild queue'.format(task)) self.system.stages[task].add(self._process_jobs[task][0][0], self._process_jobs[task][0][1]) elif rebuilds_needed is False: logger.warning("dropping {0} task, no rebuild needed.".format(task)) return None @staticmethod def process_strings(spec, strings): """ :param string spec: A string, possibly with ``format()`` style tokens in curly braces. :param dict strings: A mapping of strings to replacement values. :raises: :exc:`python:TypeError` if ``strings`` is not a dict *and* there may be a replacement string. """ if strings is None: return spec elif not isinstance(strings, dict): logger.critical('replacement content must be a dictionary.') raise TypeError if isinstance(spec, dict): out = {} for k, v in spec.items(): if '{' in v and '}' in v: try: out[k] = v.format(**strings) except KeyError as e: msg = '{0} does not have {1} suitable replacement keys.'.format(strings, e) logger.critical(msg) raise InvalidJob(msg) else: out[k] = v return out else: if '{' in spec and '}' in spec: try: spec = spec.format(**strings) except KeyError as e: msg = '{0} does not have {1} suitable replacement keys.'.format(spec, e) logger.critical(msg) raise InvalidJob(msg) return spec else: return spec @staticmethod def generate_job(spec, funcs): """ :param dict spec: A *job* specification. :param dict funcs: A dictionary mapping names (i.e. ``spec.job`` to functions.) :returns: A tuple where the first item is a callable and the second item is either a dict or a tuple of arguments, depending on the type of the ``spec.args`` value. :raises: :exc:`~err.InvalidStage` if ``spec.args`` is neither a ``dict``, ``list``, or ``tuple``. """ if isinstance(spec['args'], dict): args = spec['args'] elif isinstance(spec['args'], list): args = tuple(spec['args']) elif isinstance(spec['args'], tuple): args = spec['args'] else: raise InvalidJob('args are malformed.') if is_function(spec['job']): action = spec['job'] else: try: action = funcs[spec['job']] except KeyError: msg = "{0} not in function specification with: {0} keys".format(spec['job'], funcs.keys()) logger.critical(msg) raise InvalidJob(msg) return action, args @staticmethod def generate_shell_job(spec): """ :param dict spec: A *job* specification. :returns: A tuple where the first element is :mod:`python:subprocess` :func:`~python:subprocess.call`, and the second item is a dict ``args``, a ``dir``, and ``cmd`` keys. Takes a ``spec`` dict and returns a tuple to define a task. """ if isinstance(spec['cmd'], list): cmd_str = spec['cmd'] else: cmd_str = spec['cmd'].split() if isinstance(spec['dir'], list): base_path = os.path.sep.join(spec['dir']) if spec['dir'][0].startswith(os.path.sep): spec['dir'] = base_path else: spec['dir'] = os.path.abspath(base_path) if isinstance(spec['args'], list): cmd_str.extend(spec['args']) else: cmd_str.extend(spec['args'].split()) return subprocess.call, dict(cwd=spec['dir'], args=cmd_str) @staticmethod def generate_sequence(spec, funcs): """ :param dict spec: A *job* specification. :param dict funcs: A dictionary mapping names to callable objects. :returns: A :class:`~stages.BuildSequence()` object. :raises: An exception when ``spec`` does not have a ``task`` element or when ``task.spec`` is not a list. Process a job spec that describes a sequence of tasks and returns a :class:`~stages.BuildSequence()` object for inclusion in a :class:`~system.BuildSystem()` object. """ sequence = BuildSequence() if 'tasks' not in spec or not isinstance(spec['tasks'], list): raise InvalidStage else: for task in spec['tasks']: if 'job' in task: job, args = BuildSystemGenerator.generate_job(task, funcs) elif 'cmd' in task: job, args = BuildSystemGenerator.generate_shell_job(task) sequence.add(job, args) sequence.close() return sequence def add_task(self, name, func): """ :param string name: The identifier of a callable. :param callable func: A callable object. :raises: :exc:`~err.InvaidJob` if ``func`` is not callable. Adds a callable object to the :attr:`~system.BuildSystemGenerator.funcs` attribute with the identifier ``name``. """ if not is_function(func): logger.critical('cannot add tasks that are not callables ({0}).'.format(name)) raise InvalidJob("{0} is not callable.".format(func)) logger.debug('adding task named {0}'.format(name)) self.funcs[name] = func def ingest(self, jobs, strings=None): """ :param iterable jobs: An interable object that contains build job specifications. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. :returns: The number of jobs added to the build system. For every document """ job_count = 0 for spec in jobs: self._process_job(spec, strings) job_count += 1 logger.debug('loaded {0} jobs'.format(job_count)) return job_count def ingest_yaml(self, filename, strings=None): """ Wraps :meth:`~BuildSystemGenerator.ingest()`. :param string filename: The fully qualified path name of a :term:`YAML` file that contains a build system specification. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. For every document in the YAML file specified by ``filename``, calls :meth:`~system.BuildSystemGenerator._process_job()` to add that job to the :attr:`~system.BuildSystemGenerator.system` object. """ logger.debug('opening yaml file {0}'.format(filename)) try: with open(filename, 'r') as f: try: jobs = yaml.safe_load_all(f) except NameError: msg = 'attempting to load a yaml definition without PyYAML installed.' logger.critical(msg) raise StageRunError(msg) job_count = self.ingest(jobs, strings) logger.debug('loaded {0} jobs from {1}'.format(job_count, filename)) except IOError: logger.warning('file {0} does not exist'.format(filename)) def ingest_json(self, filename, strings=None): """ Wraps :meth:`~BuildSystemGenerator.ingest()`. :param string filename: The fully qualified path name of a :term:`JSON` file that contains a build system specification. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. For every document in the JSON file specified by ``filename``, calls :meth:`~system.BuildSystemGenerator._process_job()` to add that job to the :attr:`~system.BuildSystemGenerator.system` object. """ logger.debug('opening json file {0}'.format(filename)) try: with open(filename, 'r') as f: jobs = json.load(f) job_count = self.ingest(jobs, strings) logger.debug('loaded {0} jobs from {1}'.format(job_count, filename)) except IOError: logger.warning('file {0} does not exist'.format(filename)) def _process_stage(self, spec, spec_keys=None, strings=None): """ :param dict spec: The task specification imported from user input. :param set spec_keys: A set containing the top level keys in the set. :param dict strings: Optional. A dictionary of strings mapping to strings to use as replacement keys for spec content. :returns: A two item tuple, containing a function and a list of arguments. :raises: :exc:`~err.InvalidJob` if the schema of ``spec`` is not recognized. Based on the schema of the ``spec``, creates tasks and sequences to add to a BuildSystem using, as appropriate one of the following methods: - :meth:`~system.BuildSystemGenerator.generate_job()`, - :meth:`~system.BuildSystemGenerator.generate_shell_job()`, or - :meth:`~system.BuildSystemGenerator.generate_sequence()`. """ if spec_keys is None: spec_keys = set(spec.keys()) elif isinstance(spec_keys, set): logger.debug('using {0} for spec_keys as an optimization'.format(spec_keys)) else: logger.error('spec_keys argument {0} must be a set'.format(spec_keys)) raise InvalidJob('problem with spec_keys') if spec_keys.issuperset(set(['job', 'args'])): logger.debug('spec looks like a pure python job, processing now.') spec = self.process_strings(spec, strings) return self.generate_job(spec, self.funcs) elif spec_keys.issuperset(set(['dir', 'cmd', 'args', ])): logger.debug('spec looks like a shell job, processing now.') spec = self.process_strings(spec, strings) return self.generate_shell_job(spec) elif 'tasks' in spec_keys: logger.debug('spec looks like a shell job, processing now.') spec = self.process_strings(spec, strings) task_sequence = self.generate_sequence(spec, self.funcs) return task_sequence.run, None else: if not self.funcs: logger.warning('no functions available.') logger.critical('spec does not match existing job type, returning early') raise InvalidJob('invalid job type error') @staticmethod def get_dependency_list(spec): """ :param dict spec: A specification of a build task with a ``target``, to process. :returns: A list that specifies all specified dependencies of that task. """ if 'dependency' in spec: dependency = spec['dependency'] elif 'dep' in spec: dependency = spec['dep'] elif 'deps' in spec: dependency = spec['deps'] else: logger.warning('no dependency in {0} spec'.format(spec['target'])) return list() if isinstance(dependency, list): return dependency else: return dependency.split() def _process_dependency(self, spec): """ :param dict spec: The task specification imported from user input. Wraps :meth:`~system.BuildSystemGenerator._process_stage()` and modifies the internal strucutres associated with the dependency tree. """ job = self._process_stage(spec) dependencies = self.get_dependency_list(spec) self.specs[spec['target']] = spec if self.check.check(spec['target'], dependencies) is True: msg = 'target {0} is older than dependency {1}: adding to build queue' logger.info(msg.format(spec['target'], dependencies)) self._process_jobs[spec['target']] = (job, True) else: logger.info('rebuild not needed for {0}.') self._process_jobs[spec['target']] = (job, False) logger.debug('added {0} to dependency graph'.format(spec['target'])) self._process_tree[spec['target']] = self.get_dependency_list(spec) def _process_job(self, spec, strings=None): """ :param dict spec: A dictionary of strings that describe a build job. Processes the ``spec`` and attempts to create and add the resulting task to the build system. """ if strings is not None: for key in spec: spec[key] = spec[key].format(strings) if 'dependency' in spec or 'dep' in spec or 'deps' in spec: if ('stage' in spec and 'target' in spec): logger.error('{0} cannot have both a stage and target'.format(spec)) raise InvalidJob('stage cannot have "stage" and "target".') if 'stage' in spec and not 'target' in spec: spec['target'] = spec['stage'] del spec['stage'] if 'target' in spec: self._process_dependency(spec) logger.debug('added task to build {0}'.format(spec['target'])) return True else: if 'target' in spec and not 'stage' in spec: spec['stage'] = spec['target'] del spec['target'] if 'stage' not in spec: # put this into the BuildSystem stage to run after the dependency tree. logger.warning('job lacks stage name, adding one to "{0}", please correct.'.format(spec)) spec['stage'] = '__unspecified' job = self._process_stage(spec) if not self._stages.stage_exists(spec['stage']): logger.debug('creating new stage named {0}'.format(spec['stage'])) self._stages.new_stage(spec['stage']) self._stages.stages[spec['stage']].add(job[0], job[1]) logger.debug('added job to stage: {0}'.format(spec['stage'])) return True
class TestDependencyChecking(TestCase): @classmethod def setUp(self): self.d = DependencyChecks() self.fn_a = 'fn_a' self.fn_b = 'fn_b' @classmethod def tearDown(self): for fn in [self.fn_a, self.fn_b]: if os.path.exists(fn): os.remove(fn) def ensure_clean(self): self.assertFalse(os.path.exists(self.fn_a)) self.assertFalse(os.path.exists(self.fn_b)) def test_basic(self): self.ensure_clean() touch(self.fn_a) self.assertTrue(os.path.exists(self.fn_a)) self.assertFalse(os.path.exists(self.fn_b)) def test_basic_alt(self): self.ensure_clean() touch(self.fn_b) self.assertTrue(os.path.exists(self.fn_b)) self.assertFalse(os.path.exists(self.fn_a)) def test_default_method(self): self.ensure_clean() self.assertTrue(self.d.check_method, 'mtime') def test_setting_valid_methods(self): self.ensure_clean() for method in ['force', 'ignore', 'hash', 'mtime']: self.d.check_method = method self.assertTrue(self.d.check_method, method) self.d.check_method = 'mtime' self.assertTrue(self.d.check_method, 'mtime') def test_setting_invalid_method(self): self.ensure_clean() with self.assertRaises(DependencyCheckError): self.d.check_method = 'magic' def test_mtime_rebuild(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.assertTrue(self.d.check_method, 'mtime') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_mtime_no_rebuild(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.assertTrue(self.d.check_method, 'mtime') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_mtime_rebuild_no_target(self): self.ensure_clean() touch(self.fn_b) self.assertTrue(self.d.check_method, 'mtime') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild(self): self.ensure_clean() write(self.fn_a, 'aaa') write(self.fn_b, 'bbb') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild_ignoring_update_order(self): self.ensure_clean() write(self.fn_b, 'bbb') breath() breath() write(self.fn_a, 'aaa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_hash_no_rebuild(self): self.ensure_clean() write(self.fn_b, 'aaa') write(self.fn_a, 'aaa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_hash_rebuild_no_target(self): self.ensure_clean() write(self.fn_b, 'aa') self.d.check_method = 'hash' self.assertTrue(self.d.check_method, 'hash') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_non_existing(self): self.ensure_clean() self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_with_files(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_force_with_reversed_files(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.d.check_method = 'force' self.assertTrue(self.d.check_method, 'force') self.assertTrue(self.d.check(self.fn_a, self.fn_b)) def test_ignore_non_existing(self): self.ensure_clean() self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_ignore_with_files(self): self.ensure_clean() touch(self.fn_a) breath() touch(self.fn_b) self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b)) def test_ignore_with_reversed_files(self): self.ensure_clean() touch(self.fn_b) breath() touch(self.fn_a) self.d.check_method = 'ignore' self.assertTrue(self.d.check_method, 'ignore') self.assertFalse(self.d.check(self.fn_a, self.fn_b))