def generate_sequence(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names to callable objects.
:returns: A :class:`~stages.BuildSequence()` object.
:raises: An exception when ``spec`` does not have a ``task`` element or
when ``task.spec`` is not a list.
Process a job spec that describes a sequence of tasks and returns a
:class:`~stages.BuildSequence()` object for inclusion in a
:class:`~system.BuildSystem()` object.
"""
sequence = BuildSequence()
if 'tasks' not in spec or not isinstance(spec['tasks'], list):
raise InvalidStage
else:
for task in spec['tasks']:
if 'job' in task:
job, args = BuildSystemGenerator.generate_job(task, funcs)
elif 'cmd' in task:
job, args = BuildSystemGenerator.generate_shell_job(task)
sequence.add(job, args)
sequence.close()
return sequence
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
class TestStagesBuildSequence(StagesBuildStepTests, TestCase):
@classmethod
def setUp(self):
self.b = BuildSequence()
self.args = [ (1, 2), (2, 3), (1, 'str'), ('str', 'str') ]
self.results = []
def t_function(self, a=None, b=None):
self.results.append((a, b))
def test_running(self):
for arg in self.args:
self.b.add(self.t_function, [arg[0], arg[1]])
self.b.run()
self.assertEqual(self.results, self.args)
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
def test_init_with_stages(self):
b_one = BuildStage()
b_two = BuildSequence()
bs_one = BuildSystem()
bs_one.add_stage('one', b_one)
bs_one.add_stage('two', b_two)
bs_two = BuildSystem(bs_one)
self.assertEqual(bs_two.count(), bs_one.count())
self.assertEqual(bs_two.get_order(), bs_one.get_order())
def generate_sequence(spec, funcs):
"""
:param dict spec: A *job* specification.
:param dict funcs: A dictionary mapping names to callable objects.
:returns: A :class:`~stages.BuildSequence()` object.
:raises: An exception when ``spec`` does not have a ``task`` element or
when ``task.spec`` is not a list.
Process a job spec that describes a sequence of tasks and returns a
:class:`~stages.BuildSequence()` object for inclusion in a
:class:`~system.BuildSystem()` object.
"""
sequence = BuildSequence()
if 'tasks' not in spec or not isinstance(spec['tasks'], list):
raise InvalidStage
else:
for task in spec['tasks']:
if 'job' in task:
job, args = BuildSystemGenerator.generate_job(task, funcs)
elif 'cmd' in task:
job, args = BuildSystemGenerator.generate_shell_job(task)
sequence.add(job, args)
sequence.close()
return sequence
class TestBuildSystemSequenceGeneration(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
def test_generate_sequence(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, (1, 2, 3, 4))
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'job': dump_args_to_json_file,
'args': ['tmp', 'test']
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': dump_args_to_json_file,
'args': ['tmp', 'test']
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': subprocess.call,
'args': [1, 2, 3, 4]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_generate_sequence_shell_job(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call,
dict(cwd='/tmp', args=['test', 1, 2, 3]))
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_invalid_spec(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'job': None
}, self.funcs)
def test_invalid_spec_task(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'task': None
}, self.funcs)
def test_invalid_spec_tasks_type(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'tasks': 'string'
}, self.funcs)
def test_generated_sequence_is_closed(self):
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertTrue(ret.closed)
class TestBuildSystemSequenceGeneration(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
def test_generate_sequence(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, (1, 2, 3, 4))
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dump_args_to_json_file, 'args': ['tmp', 'test']},
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dump_args_to_json_file, 'args': ['tmp', 'test']},
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': subprocess.call, 'args': [1, 2, 3, 4] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_generate_sequence_shell_job(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, dict(cwd='/tmp', args=['test', 1, 2, 3]))
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_invalid_spec(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'job': None}, self.funcs)
def test_invalid_spec_task(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'task': None}, self.funcs)
def test_invalid_spec_tasks_type(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'tasks': 'string'}, self.funcs)
def test_generated_sequence_is_closed(self):
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertTrue(ret.closed)
def add_stage(self, name, stage=None, stage_type='stage', strict=None):
"""
:param string name: The name of a stage.
:param BuildStage stage: A :class:~stages.BuildSteps` object. Default's
to ``None``, which has the same effect as
:meth:~system.BuildSystem.new_stage()`.
:param string stage_type: Defaults to ``stage``. Either ``sequence`` (or
``seq``) or ``stage`` to determine what kind of
:class`~stages.BuildSteps()` object to instantiate if ``stage`` is
``None``
:param bool strict: Overrides the default
:attr:`~system.BuildSystem.strict` attribute. When ``True``, prevents
callers from adding duplicate stages to a :class:~system.BuildSystem`
object.
:raises: :exc:`~err.InvalidStage()` in strict mode if ``name`` or
``stage`` is malformed.
Creates a new stage and optionally populates the stage with tasks. Note
the following behaviors:
- If ``name`` exists in the current :class:~system.BuildSystem`,
``strict`` is ``False``, and ``stage`` is ``None``; then
:meth:~system.BuildSystem.add_stage()` adds the stage with ``name`` to
the current :class:~system.BuildSystem` object. This allows you to
perform the same stage multiple times in one build process.
- If ```name`` does not exist in the current
:class:~system.BuildSystem`, and ``stage`` is ``None``,
:meth:~system.BuildSystem.add_stage()` adds the stage and instantiates
a new :class`~stages.BuildSteps()` (either
:class`~stages.BuildSequence()` or :class`~stages.BuildStage()`
depending on the value of ``stage_type``) object with ``name`` to the
current :class:~system.BuildSystem` object.
:meth:~system.BuildSystem.add_stage()` must specify a ``stage_type``
if ``stage`` is ``None``. When ``strict`` is ``True`` not doing so
raises an exception.
- Adds the ``stage`` with the specified :class`~stages.BuildSteps()`
object to the :class:~system.BuildSystem` object. Raises an exception
if ``stage`` is not a :class`~stages.BuildSteps()` object.
"""
if stage is None and stage_type is 'stage':
stage = BuildStage()
logger.debug('created new (parallel) stage object')
elif stage is None and stage_type in ['seq', 'sequence']:
stage = BuildSequence()
logger.debug('created new sequence object.')
elif stage is None and stage_type not in ['stage', 'seq', 'sequence']:
logger.critical(
'no default stage object and no valid stage type named {0}.'.
format(stage_type))
self._error_or_return(
msg="must add a BuildSteps object to a build system.",
exception=InvalidStage,
strict=strict)
if isinstance(stage, BuildSteps) is False:
logger.critical('{0} is not a build step object'.format(name))
self._error_or_return(
msg="must add a BuildSteps object to a build system.",
exception=InvalidStage,
strict=strict)
else:
logger.info('appending well formed stage.')
self._stages.append(name)
self.stages[name] = stage
return True
def setUp(self):
self.jobs = [ (cpu_count, (1, 2)), (dummy_function, ('str', 'str')) ]
self.args = [ (1, 2), (2, 3), (1, 'str'), ('str', 'str') ]
self.b = BuildSequence()
self.bs = BuildSequence()
def setUp(self):
self.b = BuildSequence()
self.args = [ (1, 2), (2, 3), (1, 'str'), ('str', 'str') ]
self.results = []
