def _load_build_specs(files):
targets = []
strings = _import_strings()
for fn in files:
try:
with open(fn, 'r') as f:
if fn.endswith('json'):
tg = json.load(f)
for i in tg:
i = BuildSystemGenerator.process_strings(i, strings)
if isinstance(i, list):
for doc in i:
tg.append(i)
elif isinstance(i, dict):
targets.append(i)
else:
logger.warning(
'structure of json file {0}is unclear, ignoring file.'
.format(fn))
elif fn.endswith('yaml') or fn.endswith('yml'):
for i in yaml.safe_load_all(f):
i = BuildSystemGenerator.process_strings(i, strings)
targets.append(i)
else:
logger.warning(
'format of {0} is unclear, not parsing'.format(fn))
except OSError:
logger.warning('{0} is not readable; passing'.format(fn))
return targets
def _load_build_specs(files):
targets = []
strings = _import_strings()
for fn in files:
try:
with open(fn, 'r') as f:
if fn.endswith('json'):
tg = json.load(f)
for i in tg:
i = BuildSystemGenerator.process_strings(i, strings)
if isinstance(i, list):
for doc in i:
tg.append(i)
elif isinstance(i, dict):
targets.append(i)
else:
logger.warning('structure of json file {0}is unclear, ignoring file.'.format(fn))
elif fn.endswith('yaml') or fn.endswith('yml'):
for i in yaml.safe_load_all(f):
i = BuildSystemGenerator.process_strings(i, strings)
targets.append(i)
else:
logger.warning('format of {0} is unclear, not parsing'.format(fn))
except OSError:
logger.warning('{0} is not readable; passing'.format(fn))
return targets
def setUp(self):
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
self.bsg = BuildSystemGenerator(self.funcs)
self.bsg.check_method = 'force'
self.complex_jobs = [{
'target': 'a',
'dep': ['b', 'f', 'r'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'b',
'dep': ['c', 'l'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'c',
'dep': ['r', 'l'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'r',
'dep': [],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'f',
'dep': [],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'l',
'dep': [],
'job': 'dumb',
'args': [None, None]
}]
self.simple_jobs = [{
'target': 'a',
'dep': 'b',
'job': 'dumb',
'args': [None, None]
}, {
'target': 'b',
'dep': 'c',
'job': 'dumb',
'args': [None, None]
}, {
'target': 'c',
'dep': [],
'job': 'dumb',
'args': [None, None]
}]
def setUp(self):
self.bsg = BuildSystemGenerator()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file
}
def setUp(self):
self.bsg = BuildSystemGenerator()
self.ex_args = {
'cmd': 'go',
'cwd': '/tmp/test',
'args': ['test', 'true']
}
self.expected = (subprocess.call, self.ex_args)
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
def test_ingestion_simple(self):
self.simple_system()
self.bsg.finalize()
procedural = sorted(self.bsg.specs.keys())
second = BuildSystemGenerator(self.bsg.funcs)
second.ingest(self.simple_jobs)
second.finalize()
ingested = sorted(second.specs.keys())
self.assertEqual(procedural, ingested)
def setUp(self):
self.bsg = BuildSystemGenerator()
self.bsg.add_task('dumb', dummy_function)
self.bsg.add_task('dumb_json', dump_args_to_json_file)
self.bsg.add_task('dumb_json_newline',
dump_args_to_json_file_with_newlines)
self.bsg.add_task('shell', subprocess.call)
self.funcs = {
'dumb': dummy_function,
'dump_json_newline': dump_args_to_json_file_with_newlines,
'shell': subprocess.call,
'dumb_json': dump_args_to_json_file
}
self.strings = {'is': 'is not', 'can': 'cannot'}
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
def stages(jobs, stages, file, check):
"""
Main public function to generate and run a
:class:`~system.BuildSystemGenerator()` build system.
"""
if os.path.isdir('buildc') or os.path.exists('buildc.py'):
try:
from buildc import functions
except ImportError:
from buildc import funcs as functions
else:
functions = None
else:
functions = None
strings = _import_strings()
bsg = BuildSystemGenerator(functions)
bsg.check_method = check
if functions is None:
logger.info('no python functions pre-loaded')
for fn in file:
if fn.endswith('json'):
bsg.ingest_json(fn, strings)
elif fn.endswith('yaml') or fn.endswith('yml'):
bsg.ingest_yaml(fn, strings)
else:
logger.warning('format of {0} is unclear, not parsing'.format(fn))
bsg.finalize()
bsg.system.workers(jobs)
if not stages:
bsg.system.run()
else:
bsg = narrow_buildsystem(stages, bsg.system)
bsg.system.workers(jobs)
bsg.run()
def setUp(self):
self.bsg = BuildSystemGenerator()
self.bsg.add_task('dumb', dummy_function)
self.bsg.add_task('dumb_json', dump_args_to_json_file)
self.bsg.add_task('dumb_json_newline', dump_args_to_json_file_with_newlines)
self.bsg.add_task('shell', subprocess.call)
self.funcs = { 'dumb': dummy_function,
'dump_json_newline': dump_args_to_json_file_with_newlines,
'shell': subprocess.call,
'dumb_json': dump_args_to_json_file }
self.strings = { 'is': 'is not',
'can': 'cannot' }
def test_ingestion_simple(self):
self.simple_system()
self.bsg.finalize()
procedural = sorted(self.bsg.specs.keys())
second = BuildSystemGenerator(self.bsg.funcs)
second.ingest(self.simple_jobs)
second.finalize()
ingested = sorted(second.specs.keys())
self.assertEqual(procedural, ingested)
class TestBuildSystemGeneratorShellJob(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.ex_args = { 'cmd': 'go',
'cwd': '/tmp/test',
'args': ['test', 'true']}
self.expected = (subprocess.call, self.ex_args)
def assertions_abs_path(self, ret):
self.assertEquals(ret[0], self.expected[0])
self.assertEquals(self.ex_args['cmd'], ret[1]['args'][0])
self.assertEquals(self.ex_args['cwd'], ret[1]['cwd'])
self.assertEquals(self.ex_args['args'], ret[1]['args'][1:])
def assertions_rel_path(self, ret):
ex_path = os.path.join(os.path.abspath(self.ex_args['cwd'][1:]))
self.assertEquals(ret[0], self.expected[0])
self.assertEquals(self.ex_args['cmd'], ret[1]['args'][0])
self.assertEquals(ex_path, ret[1]['cwd'])
self.assertEquals(self.ex_args['args'], ret[1]['args'][1:])
def test_generate_shell_job_list_args(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': ['test', 'true'], 'dir': '/tmp/test'})
self.assertions_abs_path(ret)
def test_generate_shell_job_string_args(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': 'test true', 'dir': '/tmp/test'})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': 'test true', 'dir': ['/tmp', 'test']})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path_list_args(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': ['test', 'true'], 'dir': ['/tmp', 'test']})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path_rel_path_list_args(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': ['test', 'true'], 'dir': ['tmp', 'test']})
self.assertions_rel_path(ret)
def test_generate_shell_job_list_path_rel_path_string_args(self):
ret = self.bsg.generate_shell_job({ 'cmd': 'go', 'args': 'test true', 'dir': ['tmp', 'test']})
self.assertions_rel_path(ret)
class TestBuildSystemSequenceGeneration(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
def test_generate_sequence(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, (1, 2, 3, 4))
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dump_args_to_json_file, 'args': ['tmp', 'test']},
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': dump_args_to_json_file, 'args': ['tmp', 'test']},
{ 'job': dummy_function, 'args': [1, 2 ] },
{ 'job': subprocess.call, 'args': [1, 2, 3, 4] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_generate_sequence_shell_job(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, dict(cwd='/tmp', args=['test', 1, 2, 3]))
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_invalid_spec(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'job': None}, self.funcs)
def test_invalid_spec_task(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'task': None}, self.funcs)
def test_invalid_spec_tasks_type(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({'stage': 'cmd', 'tasks': 'string'}, self.funcs)
def test_generated_sequence_is_closed(self):
spec = { 'tasks': [ { 'job': dummy_function, 'args': [1, 2 ] },
{ 'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3] } ],
'stage': 'test'}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertTrue(ret.closed)
class TestBuildSystemSequenceGeneration(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.sequence = BuildSequence()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
def test_generate_sequence(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(dump_args_to_json_file, ('tmp', 'test'))
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call, (1, 2, 3, 4))
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'job': dump_args_to_json_file,
'args': ['tmp', 'test']
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': dump_args_to_json_file,
'args': ['tmp', 'test']
}, {
'job': dummy_function,
'args': [1, 2]
}, {
'job': subprocess.call,
'args': [1, 2, 3, 4]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_generate_sequence_shell_job(self):
self.sequence.add(dummy_function, (1, 2))
self.sequence.add(subprocess.call,
dict(cwd='/tmp', args=['test', 1, 2, 3]))
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertEqual(ret.stage, self.sequence.stage)
def test_invalid_spec(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'job': None
}, self.funcs)
def test_invalid_spec_task(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'task': None
}, self.funcs)
def test_invalid_spec_tasks_type(self):
with self.assertRaises(InvalidStage):
self.bsg.generate_sequence({
'stage': 'cmd',
'tasks': 'string'
}, self.funcs)
def test_generated_sequence_is_closed(self):
spec = {
'tasks': [{
'job': dummy_function,
'args': [1, 2]
}, {
'cmd': "test",
'dir': '/tmp',
'args': [1, 2, 3]
}],
'stage':
'test'
}
ret = self.bsg.generate_sequence(spec, self.funcs)
self.assertTrue(ret.closed)
class TestBuildGeneratorProcessStageSpecs(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.bsg.add_task('dumb', dummy_function)
self.bsg.add_task('dumb_json', dump_args_to_json_file)
self.bsg.add_task('dumb_json_newline', dump_args_to_json_file_with_newlines)
self.bsg.add_task('shell', subprocess.call)
self.funcs = { 'dumb': dummy_function,
'dump_json_newline': dump_args_to_json_file_with_newlines,
'shell': subprocess.call,
'dumb_json': dump_args_to_json_file }
self.strings = { 'is': 'is not',
'can': 'cannot' }
def test_process_stage_python_job(self):
spec = { 'job': 'dumb',
'args': [1,2,3],
'stage': 'test',
'msg': 'wtf'}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_job(spec, self.funcs)
self.assertEqual(ret, expected)
def test_process_stage_python_job_with_replacement(self):
spec = { 'job': 'dumb',
'args': [1,2,3],
'stage': 'test',
'msg': 'wtf {can} this be or {is}'}
ret = self.bsg._process_stage(spec, strings=self.strings)
ex_spec = self.bsg.process_strings(spec, self.strings)
expected = self.bsg.generate_job(ex_spec, self.funcs)
self.assertEqual(ret, expected)
def test_process_stage_shell_job(self):
spec = { 'dir': '/tmp',
'args': [1,2,3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf'}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_shell_job(spec)
self.assertEqual(ret, expected)
def test_process_stage_shell_job_with_replacement(self):
spec = { 'dir': '/tmp',
'args': [1,2,3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'}
ex_spec = self.bsg.process_strings(spec, self.strings)
ret = self.bsg._process_stage(spec, strings=self.strings)
expected = self.bsg.generate_shell_job(ex_spec)
self.assertEqual(ret, expected)
def test_process_stage_task_sequence(self):
spec = { 'stage': 'test',
'tasks': [
{ 'dir': '/tmp',
'args': [1,2,3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'},
{ 'job': 'dumb',
'args': [1,2,3],
'stage': 'test',
'msg': 'wtf' }
],
'msg': 'wtf {can} this be or {is}'}
ret = self.bsg._process_stage(spec)
expected = (self.bsg.generate_sequence(spec, self.funcs).run, None)
self.assertEqual(str(ret)[:-18], str(expected)[:-18])
def test_process_stage_task_sequence_replacement(self):
spec = { 'stage': 'test',
'tasks': [
{ 'dir': '/tmp',
'args': [1,2,3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'},
{ 'job': 'dumb',
'args': [1,2,3],
'stage': 'test',
'msg': 'wtf' }
],
'msg': 'wtf {can} this be or {is}'}
ex_spec = self.bsg.process_strings(spec, self.strings)
ret = self.bsg._process_stage(spec, strings=self.strings)
expected = (self.bsg.generate_sequence(ex_spec, self.funcs).run, None)
self.assertEqual(str(ret)[:-18], str(expected)[:-18])
def test_invalid_spec(self):
spec = { 'is': 'is not',
'can': 'cannot' }
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec)
def test_invalid_spec_with_keys(self):
spec_keys = set(['stage', 'task'])
spec = { 'stage': 'is not',
'test': 'cannot',
'msg': 'stage' }
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec, spec_keys=spec_keys)
def test_invalid_spec_with_keys(self):
spec_keys = set(['stage', 'task'])
spec = { 'is': 'is not',
'can': 'cannot' }
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec, spec_keys=spec_keys)
def test_process_stage_shell_job_with_keys(self):
spec_keys = set(['dir', 'cmd', 'args', 'stage'])
spec = { 'dir': '/tmp',
'args': [1,2,3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf'}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_shell_job(spec)
self.assertEqual(ret, expected)
class TestBuildSystemGeneratorShellJob(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.ex_args = {
'cmd': 'go',
'cwd': '/tmp/test',
'args': ['test', 'true']
}
self.expected = (subprocess.call, self.ex_args)
def assertions_abs_path(self, ret):
self.assertEquals(ret[0], self.expected[0])
self.assertEquals(self.ex_args['cmd'], ret[1]['args'][0])
self.assertEquals(self.ex_args['cwd'], ret[1]['cwd'])
self.assertEquals(self.ex_args['args'], ret[1]['args'][1:])
def assertions_rel_path(self, ret):
ex_path = os.path.join(os.path.abspath(self.ex_args['cwd'][1:]))
self.assertEquals(ret[0], self.expected[0])
self.assertEquals(self.ex_args['cmd'], ret[1]['args'][0])
self.assertEquals(ex_path, ret[1]['cwd'])
self.assertEquals(self.ex_args['args'], ret[1]['args'][1:])
def test_generate_shell_job_list_args(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': ['test', 'true'],
'dir': '/tmp/test'
})
self.assertions_abs_path(ret)
def test_generate_shell_job_string_args(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': 'test true',
'dir': '/tmp/test'
})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': 'test true',
'dir': ['/tmp', 'test']
})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path_list_args(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': ['test', 'true'],
'dir': ['/tmp', 'test']
})
self.assertions_abs_path(ret)
def test_generate_shell_job_list_path_rel_path_list_args(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': ['test', 'true'],
'dir': ['tmp', 'test']
})
self.assertions_rel_path(ret)
def test_generate_shell_job_list_path_rel_path_string_args(self):
ret = self.bsg.generate_shell_job({
'cmd': 'go',
'args': 'test true',
'dir': ['tmp', 'test']
})
self.assertions_rel_path(ret)
class TestBuildSystemGenerator(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file }
def test_initiation_funcs_empty(self):
self.assertEqual(self.bsg.funcs, {})
def test_initiation_funcs_initialized(self):
func = { 'a': dummy_function }
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, func)
def test_initiation_funcs_initialized_tuple(self):
func = ('a', dummy_function)
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, {})
def test_initiation_internal_stages(self):
self.assertTrue(isinstance(self.bsg._stages, BuildSystem))
def test_initiation_dependency_checking_object(self):
self.assertTrue(isinstance(self.bsg.check, DependencyChecks))
def test_initiation_empty_system(self):
self.assertEqual(self.bsg.system, None)
def test_initiation_of_tree_structures(self):
self.assertTrue(isinstance(self.bsg._process_jobs, dict))
self.assertTrue(isinstance(self.bsg._process_tree, dict))
def test_initial_object_is_open(self):
self.assertFalse(self.bsg._final)
def test_check_method_correct_setting(self):
self.assertEqual(self.bsg.check._check, self.bsg.check_method)
def test_check_method_correct_value(self):
self.assertTrue(self.bsg.check_method in self.bsg.check.checks)
def test_check_method_resistance(self):
o = self.bsg.check_method
self.bsg.check_method = '_abcdefg'
self.assertEqual(o, self.bsg.check_method)
def test_check_method_change_success(self):
o = self.bsg.check_method
self.bsg.check_method = 'hash'
self.assertNotEqual(o, self.bsg.check_method)
def test_process_strings_no_token(self):
string = 'this string has no replacement tokens'
self.assertEqual(self.bsg.process_strings(string, {}), string)
def test_process_replacement_strings_not_dict(self):
with self.assertRaises(TypeError):
self.bsg.process_strings(['a'], 'f{o}o')
def test_process_successful_replacement(self):
new = 'this is not a car.'
old = 'this {is} a car.'
p = self.bsg.process_strings(old, { 'is': 'is not'})
self.assertEqual(p, new)
def test_process_irrelevant_spec(self):
old = 'this {is} a car.'
with self.assertRaises(InvalidJob):
p = self.bsg.process_strings(old,{ "isn't'": 'is not'})
def test_process_half_relevant_spec(self):
old = 'this {works} or {not}'
new = 'this wwworks or {not}'
strings = {'works': 'wwworks', 'car': 'cccar'}
with self.assertRaises(InvalidJob):
self.assertEqual(self.bsg.process_strings(old, strings), new)
def test_process_replacements_in_dict(self):
old = {'message': 'this {car} {works}'}
new = {'message': 'this cccar wwworks'}
strings = {'works': 'wwworks', 'car': 'cccar'}
ret = self.bsg.process_strings(old, strings)
self.assertEqual(ret, new)
alt = { 'message': old['message'].format(**strings)}
self.assertEqual(alt, new)
def test_generate_job_list(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': 'dumb', 'args': [1, 2]}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_tuple(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_dict(self):
args = {'a': 1, 'b': 2}
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_malformed_args(self):
args = BuildSystem()
with self.assertRaises(InvalidJob):
self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
def test_generate_job_non_extant_function(self):
args = (1, 2)
with self.assertRaises(InvalidJob):
self.bsg.generate_job({'job': 'Nope', 'args': args}, self.funcs)
def test_generate_job_list_func_callable(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': dummy_function, 'args': [1, 2]}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_tuple_func_callable(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': dummy_function, 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_dict_func_callable(self):
args = {'a': 1, 'b': 2}
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': dummy_function, 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_malformed_args_func_callable(self):
args = BuildSystem()
with self.assertRaises(InvalidJob):
self.bsg.generate_job({'job': dummy_function, 'args': args}, self.funcs)
def test_adding_task_to_funcs(self):
self.assertEqual(self.bsg.funcs, {})
self.bsg.add_task('foo', dummy_function)
self.assertEqual(self.bsg.funcs, {'foo': dummy_function})
def test_adding_task_to_funcs_invalid_job(self):
self.assertEqual(self.bsg.funcs, {})
with self.assertRaises(InvalidJob):
self.bsg.add_task('foo', BuildSystemGenerator)
self.assertEqual(self.bsg.funcs, {})
# TODO tests for ingest_yaml
# TODO tests for ingest_json
def test_dependency_string_full_name(self):
spec = { 'dependency': 'a b c d',
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_string_shortest(self):
spec = { 'dep': 'a b c d',
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_string_short_alt(self):
spec = { 'deps': 'a b c d',
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_full_name(self):
spec = { 'dependency': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_shortest(self):
spec = { 'dep': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_short_alt(self):
spec = { 'deps': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha' }
expected = [ 'a', 'b', 'c', 'd' ]
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_process_job_with_stage_and_target(self):
spec = {
'target': 'other',
'dependency': 'string',
'job': 'dumb',
'args': None,
'stage': 'one',
}
with self.assertRaises(InvalidJob):
self.bsg._process_job(spec)
def test_process_dependency_list(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = { 'deps': ['a', 'b', 'c', 'd'],
'stage': 'test',
'target': '/tmp/files0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.bsg._process_dependency(spec)
self.assertEqual(spec['deps'], self.bsg._process_tree[spec['target']])
def test_process_dependency_string(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = { 'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files1',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.bsg._process_dependency(spec)
self.assertEqual(spec['deps'].split(),
self.bsg._process_tree[spec['target']])
def test_process_dependency_rebuild_needed(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = { 'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files2',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.bsg._process_dependency(spec)
self.assertTrue(self.bsg._process_jobs[spec['target']][1])
def test_process_dependency_rebuild_not_needed(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'ignore'
spec = { 'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.bsg._process_dependency(spec)
self.assertFalse(self.bsg._process_jobs[spec['target']][1])
def test_job_processing_dep_target(self):
spec = { 'target': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_no_dep_target(self):
spec = { 'target': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_dep_stage(self):
spec = { 'stage': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_no_dep_stage(self):
spec = { 'stage': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_invalid_with_stage_and_target(self):
spec = { 'stage': 'test',
'target': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha' }
with self.assertRaises(InvalidJob):
self.bsg._process_job(spec)
class TestBuildSystemGenerator(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file
}
def test_initiation_funcs_empty(self):
self.assertEqual(self.bsg.funcs, {})
def test_initiation_funcs_initialized(self):
func = {'a': dummy_function}
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, func)
def test_initiation_funcs_initialized_tuple(self):
func = ('a', dummy_function)
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, {})
def test_initiation_internal_stages(self):
self.assertTrue(isinstance(self.bsg._stages, BuildSystem))
def test_initiation_dependency_checking_object(self):
self.assertTrue(isinstance(self.bsg.check, DependencyChecks))
def test_initiation_empty_system(self):
self.assertEqual(self.bsg.system, None)
def test_initiation_of_tree_structures(self):
self.assertTrue(isinstance(self.bsg._process_jobs, dict))
self.assertTrue(isinstance(self.bsg._process_tree, dict))
def test_initial_object_is_open(self):
self.assertFalse(self.bsg._final)
def test_check_method_correct_setting(self):
self.assertEqual(self.bsg.check._check, self.bsg.check_method)
def test_check_method_correct_value(self):
self.assertTrue(self.bsg.check_method in self.bsg.check.checks)
def test_check_method_resistance(self):
o = self.bsg.check_method
self.bsg.check_method = '_abcdefg'
self.assertEqual(o, self.bsg.check_method)
def test_check_method_change_success(self):
o = self.bsg.check_method
self.bsg.check_method = 'hash'
self.assertNotEqual(o, self.bsg.check_method)
def test_process_strings_no_token(self):
string = 'this string has no replacement tokens'
self.assertEqual(self.bsg.process_strings(string, {}), string)
def test_process_replacement_strings_not_dict(self):
with self.assertRaises(TypeError):
self.bsg.process_strings(['a'], 'f{o}o')
def test_process_successful_replacement(self):
new = 'this is not a car.'
old = 'this {is} a car.'
p = self.bsg.process_strings(old, {'is': 'is not'})
self.assertEqual(p, new)
def test_process_irrelevant_spec(self):
old = 'this {is} a car.'
with self.assertRaises(InvalidJob):
p = self.bsg.process_strings(old, {"isn't'": 'is not'})
def test_process_half_relevant_spec(self):
old = 'this {works} or {not}'
new = 'this wwworks or {not}'
strings = {'works': 'wwworks', 'car': 'cccar'}
with self.assertRaises(InvalidJob):
self.assertEqual(self.bsg.process_strings(old, strings), new)
def test_process_replacements_in_dict(self):
old = {'message': 'this {car} {works}'}
new = {'message': 'this cccar wwworks'}
strings = {'works': 'wwworks', 'car': 'cccar'}
ret = self.bsg.process_strings(old, strings)
self.assertEqual(ret, new)
alt = {'message': old['message'].format(**strings)}
self.assertEqual(alt, new)
def test_generate_job_list(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({
'job': 'dumb',
'args': [1, 2]
}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_tuple(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_dict(self):
args = {'a': 1, 'b': 2}
expected = (dummy_function, args)
ret = self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_malformed_args(self):
args = BuildSystem()
with self.assertRaises(InvalidJob):
self.bsg.generate_job({'job': 'dumb', 'args': args}, self.funcs)
def test_generate_job_non_extant_function(self):
args = (1, 2)
with self.assertRaises(InvalidJob):
self.bsg.generate_job({'job': 'Nope', 'args': args}, self.funcs)
def test_generate_job_list_func_callable(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({
'job': dummy_function,
'args': [1, 2]
}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_tuple_func_callable(self):
args = (1, 2)
expected = (dummy_function, args)
ret = self.bsg.generate_job({
'job': dummy_function,
'args': args
}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_dict_func_callable(self):
args = {'a': 1, 'b': 2}
expected = (dummy_function, args)
ret = self.bsg.generate_job({
'job': dummy_function,
'args': args
}, self.funcs)
self.assertEqual(expected, ret)
def test_generate_job_malformed_args_func_callable(self):
args = BuildSystem()
with self.assertRaises(InvalidJob):
self.bsg.generate_job({
'job': dummy_function,
'args': args
}, self.funcs)
def test_adding_task_to_funcs(self):
self.assertEqual(self.bsg.funcs, {})
self.bsg.add_task('foo', dummy_function)
self.assertEqual(self.bsg.funcs, {'foo': dummy_function})
def test_adding_task_to_funcs_invalid_job(self):
self.assertEqual(self.bsg.funcs, {})
with self.assertRaises(InvalidJob):
self.bsg.add_task('foo', BuildSystemGenerator)
self.assertEqual(self.bsg.funcs, {})
# TODO tests for ingest_yaml
# TODO tests for ingest_json
def test_dependency_string_full_name(self):
spec = {
'dependency': 'a b c d',
'target': '/tmp/files',
'msg': 'alpha'
}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_string_shortest(self):
spec = {'dep': 'a b c d', 'target': '/tmp/files', 'msg': 'alpha'}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_string_short_alt(self):
spec = {'deps': 'a b c d', 'target': '/tmp/files', 'msg': 'alpha'}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_full_name(self):
spec = {
'dependency': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha'
}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_shortest(self):
spec = {
'dep': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha'
}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_dependency_list_short_alt(self):
spec = {
'deps': ['a', 'b', 'c', 'd'],
'target': '/tmp/files',
'msg': 'alpha'
}
expected = ['a', 'b', 'c', 'd']
self.assertEqual(self.bsg.get_dependency_list(spec), expected)
def test_process_job_with_stage_and_target(self):
spec = {
'target': 'other',
'dependency': 'string',
'job': 'dumb',
'args': None,
'stage': 'one',
}
with self.assertRaises(InvalidJob):
self.bsg._process_job(spec)
def test_process_dependency_list(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = {
'deps': ['a', 'b', 'c', 'd'],
'stage': 'test',
'target': '/tmp/files0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.bsg._process_dependency(spec)
self.assertEqual(spec['deps'], self.bsg._process_tree[spec['target']])
def test_process_dependency_string(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = {
'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files1',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.bsg._process_dependency(spec)
self.assertEqual(spec['deps'].split(),
self.bsg._process_tree[spec['target']])
def test_process_dependency_rebuild_needed(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'force'
spec = {
'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files2',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.bsg._process_dependency(spec)
self.assertTrue(self.bsg._process_jobs[spec['target']][1])
def test_process_dependency_rebuild_not_needed(self):
self.assertEqual(self.bsg._process_jobs, {})
self.assertEqual(self.bsg._process_tree, {})
self.bsg.check.check_method = 'ignore'
spec = {
'deps': 'a b c d',
'stage': 'test',
'target': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.bsg._process_dependency(spec)
self.assertFalse(self.bsg._process_jobs[spec['target']][1])
def test_job_processing_dep_target(self):
spec = {
'target': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_no_dep_target(self):
spec = {
'target': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_dep_stage(self):
spec = {
'stage': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_no_dep_stage(self):
spec = {
'stage': '/tmp/files3',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
self.assertTrue(self.bsg._process_job(spec))
def test_job_processing_invalid_with_stage_and_target(self):
spec = {
'stage': 'test',
'target': '/tmp/files3',
'dependency': ['a', 'b'],
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b'],
'msg': 'alpha'
}
with self.assertRaises(InvalidJob):
self.bsg._process_job(spec)
def setUp(self):
self.bsg = BuildSystemGenerator()
def setUp(self):
self.bsg = BuildSystemGenerator()
def test_initiation_funcs_initialized(self):
func = {'a': dummy_function}
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, func)
def stages(jobs, stages, file, check):
"""
Main public function to generate and run a
:class:`~system.BuildSystemGenerator()` build system.
"""
if os.path.isdir('buildc') or os.path.exists('buildc.py'):
try:
from buildc import functions
except ImportError:
from buildc import funcs as functions
else:
functions = None
else:
functions = None
strings = _import_strings()
bsg = BuildSystemGenerator(functions)
bsg.check_method = check
if functions is None:
logger.info('no python functions pre-loaded')
for fn in file:
if fn.endswith('json'):
bsg.ingest_json(fn, strings)
elif fn.endswith('yaml') or fn.endswith('yml'):
bsg.ingest_yaml(fn, strings)
else:
logger.warning('format of {0} is unclear, not parsing'.format(fn))
bsg.finalize()
bsg.system.workers(jobs)
if not stages:
bsg.system.run()
else:
bsg = narrow_buildsystem(stages, bsg.system)
bsg.system.workers(jobs)
bsg.run()
def setUp(self):
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
self.bsg = BuildSystemGenerator(self.funcs)
self.bsg.check_method = 'force'
self.complex_jobs = [
{
'target': 'a',
'dep': ['b', 'f', 'r'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'b',
'dep': ['c', 'l'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'c',
'dep': ['r', 'l'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'r',
'dep': [],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'f',
'dep': [],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'l',
'dep': [],
'job': 'dumb',
'args': [None, None]
}
]
self.simple_jobs = [
{
'target': 'a',
'dep': 'b',
'job': 'dumb',
'args': [None, None]
},
{
'target': 'b',
'dep': 'c',
'job': 'dumb',
'args': [None, None]
},
{
'target': 'c',
'dep': [],
'job': 'dumb',
'args': [None, None]
}
]
def setUp(self):
self.bsg = BuildSystemGenerator()
self.funcs = { 'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file }
class TestBuildGeneratorProcessStageSpecs(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
self.bsg.add_task('dumb', dummy_function)
self.bsg.add_task('dumb_json', dump_args_to_json_file)
self.bsg.add_task('dumb_json_newline',
dump_args_to_json_file_with_newlines)
self.bsg.add_task('shell', subprocess.call)
self.funcs = {
'dumb': dummy_function,
'dump_json_newline': dump_args_to_json_file_with_newlines,
'shell': subprocess.call,
'dumb_json': dump_args_to_json_file
}
self.strings = {'is': 'is not', 'can': 'cannot'}
def test_process_stage_python_job(self):
spec = {
'job': 'dumb',
'args': [1, 2, 3],
'stage': 'test',
'msg': 'wtf'
}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_job(spec, self.funcs)
self.assertEqual(ret, expected)
def test_process_stage_python_job_with_replacement(self):
spec = {
'job': 'dumb',
'args': [1, 2, 3],
'stage': 'test',
'msg': 'wtf {can} this be or {is}'
}
ret = self.bsg._process_stage(spec, strings=self.strings)
ex_spec = self.bsg.process_strings(spec, self.strings)
expected = self.bsg.generate_job(ex_spec, self.funcs)
self.assertEqual(ret, expected)
def test_process_stage_shell_job(self):
spec = {
'dir': '/tmp',
'args': [1, 2, 3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf'
}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_shell_job(spec)
self.assertEqual(ret, expected)
def test_process_stage_shell_job_with_replacement(self):
spec = {
'dir': '/tmp',
'args': [1, 2, 3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'
}
ex_spec = self.bsg.process_strings(spec, self.strings)
ret = self.bsg._process_stage(spec, strings=self.strings)
expected = self.bsg.generate_shell_job(ex_spec)
self.assertEqual(ret, expected)
def test_process_stage_task_sequence(self):
spec = {
'stage':
'test',
'tasks': [{
'dir': '/tmp',
'args': [1, 2, 3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'
}, {
'job': 'dumb',
'args': [1, 2, 3],
'stage': 'test',
'msg': 'wtf'
}],
'msg':
'wtf {can} this be or {is}'
}
ret = self.bsg._process_stage(spec)
expected = (self.bsg.generate_sequence(spec, self.funcs).run, None)
self.assertEqual(str(ret)[:-18], str(expected)[:-18])
def test_process_stage_task_sequence_replacement(self):
spec = {
'stage':
'test',
'tasks': [{
'dir': '/tmp',
'args': [1, 2, 3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf {can} this be or {is}'
}, {
'job': 'dumb',
'args': [1, 2, 3],
'stage': 'test',
'msg': 'wtf'
}],
'msg':
'wtf {can} this be or {is}'
}
ex_spec = self.bsg.process_strings(spec, self.strings)
ret = self.bsg._process_stage(spec, strings=self.strings)
expected = (self.bsg.generate_sequence(ex_spec, self.funcs).run, None)
self.assertEqual(str(ret)[:-18], str(expected)[:-18])
def test_invalid_spec(self):
spec = {'is': 'is not', 'can': 'cannot'}
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec)
def test_invalid_spec_with_keys(self):
spec_keys = set(['stage', 'task'])
spec = {'stage': 'is not', 'test': 'cannot', 'msg': 'stage'}
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec, spec_keys=spec_keys)
def test_invalid_spec_with_keys(self):
spec_keys = set(['stage', 'task'])
spec = {'is': 'is not', 'can': 'cannot'}
with self.assertRaises(InvalidJob):
self.bsg._process_stage(spec, spec_keys=spec_keys)
def test_process_stage_shell_job_with_keys(self):
spec_keys = set(['dir', 'cmd', 'args', 'stage'])
spec = {
'dir': '/tmp',
'args': [1, 2, 3],
'cmd': 'cat',
'stage': 'test',
'msg': 'wtf'
}
ret = self.bsg._process_stage(spec)
expected = self.bsg.generate_shell_job(spec)
self.assertEqual(ret, expected)
def setUp(self):
self.bsg = BuildSystemGenerator()
self.ex_args = { 'cmd': 'go',
'cwd': '/tmp/test',
'args': ['test', 'true']}
self.expected = (subprocess.call, self.ex_args)
class TestBuildSystemGeneratorFunctional(TestCase):
@classmethod
def setUp(self):
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call
}
self.bsg = BuildSystemGenerator(self.funcs)
self.bsg.check_method = 'force'
self.complex_jobs = [{
'target': 'a',
'dep': ['b', 'f', 'r'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'b',
'dep': ['c', 'l'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'c',
'dep': ['r', 'l'],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'r',
'dep': [],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'f',
'dep': [],
'job': 'dumb',
'args': [None, None]
}, {
'target': 'l',
'dep': [],
'job': 'dumb',
'args': [None, None]
}]
self.simple_jobs = [{
'target': 'a',
'dep': 'b',
'job': 'dumb',
'args': [None, None]
}, {
'target': 'b',
'dep': 'c',
'job': 'dumb',
'args': [None, None]
}, {
'target': 'c',
'dep': [],
'job': 'dumb',
'args': [None, None]
}]
def simple_system(self):
for i in self.simple_jobs:
self.bsg._process_job(i)
def complex_system(self):
for i in self.complex_jobs:
self.bsg._process_job(i)
def test_dependency_ordering_one(self):
self.simple_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process, ['a', 'b', 'c'])
def test_graph_assemblage_simple_comp(self):
self.simple_system()
self.bsg.finalize()
self.assertTrue('a' in self.bsg._process_tree)
self.assertTrue('b' in self.bsg._process_tree)
self.assertTrue('c' in self.bsg._process_tree)
def test_graph_assemblage_simple_comp(self):
self.simple_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process_tree['a'], ['b'])
self.assertEqual(self.bsg._process_tree['b'], ['c'])
self.assertEqual(self.bsg._process_tree['c'], [])
def test_dependency_ordering_complex(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process, ['a', 'f', 'b', 'c', 'l', 'r'])
def test_dependency_ordering_complex_comp(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process_tree['a'], ['b', 'f', 'r'])
self.assertEqual(self.bsg._process_tree['b'], ['c', 'l'])
self.assertEqual(self.bsg._process_tree['c'], ['r', 'l'])
self.assertEqual(self.bsg._process_tree['r'], [])
self.assertEqual(self.bsg._process_tree['f'], [])
self.assertEqual(self.bsg._process_tree['l'], [])
def test_job_grouping(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg.system._stages[0], 'r')
job = (self.funcs['dumb'], (None, None))
for i in self.bsg.system.stages['r'].stage:
self.assertEqual(i, job)
def test_run_without_errors_complex(self):
self.complex_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_run_without_errors_simple(self):
self.simple_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_run_without_errors_combined(self):
self.complex_system()
self.simple_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_ingestion_simple(self):
self.simple_system()
self.bsg.finalize()
procedural = sorted(self.bsg.specs.keys())
second = BuildSystemGenerator(self.bsg.funcs)
second.ingest(self.simple_jobs)
second.finalize()
ingested = sorted(second.specs.keys())
self.assertEqual(procedural, ingested)
def test_buildsystem_narrowing_complex(self):
self.complex_system()
self.bsg.finalize()
new_bsg = narrow_buildsystem('c', self.bsg)
for i in ['c', 'r', 'l']:
self.assertTrue(i in new_bsg._process_tree)
self.assertTrue(len(new_bsg._process_tree) == 3)
class TestBuildSystemGeneratorFinalize(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
def test_add_tasks_to_stage_invalid(self):
self.assertEqual(
None,
self.bsg._add_tasks_to_stage(False, 1, 3, {'a': 'b'}, ['a', 'b']))
def test_finalize_on_empty(self):
self.assertFalse(self.bsg._final)
self.assertEqual(None, self.bsg.system)
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_finalize_second_time(self):
self.bsg._final = True
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_finalize_with_non_clean_system(self):
self.assertFalse(self.bsg._final)
self.assertEqual(None, self.bsg.system)
self.bsg.system = True
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_final_toggles_final_state(self):
self.assertFalse(self.bsg._final)
self.bsg._process_job({
'stage': 'test0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b']
})
self.bsg.finalize()
self.assertTrue(self.bsg._final)
def test_final_creates_build_system_object(self):
self.assertFalse(self.bsg._final)
self.bsg._process_job({
'stage': 'test0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b']
})
self.bsg.finalize()
self.assertTrue(isinstance(self.bsg.system, BuildSystem))
class TestBuildSystemGeneratorFunctional(TestCase):
@classmethod
def setUp(self):
self.funcs = {
'dumb': dummy_function,
'dump_new': dump_args_to_json_file_with_newlines,
'dump': dump_args_to_json_file,
'shell': subprocess.call }
self.bsg = BuildSystemGenerator(self.funcs)
self.bsg.check_method = 'force'
self.complex_jobs = [
{
'target': 'a',
'dep': ['b', 'f', 'r'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'b',
'dep': ['c', 'l'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'c',
'dep': ['r', 'l'],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'r',
'dep': [],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'f',
'dep': [],
'job': 'dumb',
'args': [None, None]
},
{
'target': 'l',
'dep': [],
'job': 'dumb',
'args': [None, None]
}
]
self.simple_jobs = [
{
'target': 'a',
'dep': 'b',
'job': 'dumb',
'args': [None, None]
},
{
'target': 'b',
'dep': 'c',
'job': 'dumb',
'args': [None, None]
},
{
'target': 'c',
'dep': [],
'job': 'dumb',
'args': [None, None]
}
]
def simple_system(self):
for i in self.simple_jobs:
self.bsg._process_job(i)
def complex_system(self):
for i in self.complex_jobs:
self.bsg._process_job(i)
def test_dependency_ordering_one(self):
self.simple_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process, ['a', 'b', 'c'])
def test_graph_assemblage_simple_comp(self):
self.simple_system()
self.bsg.finalize()
self.assertTrue('a' in self.bsg._process_tree)
self.assertTrue('b' in self.bsg._process_tree)
self.assertTrue('c' in self.bsg._process_tree)
def test_graph_assemblage_simple_comp(self):
self.simple_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process_tree['a'], ['b'])
self.assertEqual(self.bsg._process_tree['b'], ['c'])
self.assertEqual(self.bsg._process_tree['c'], [])
def test_dependency_ordering_complex(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process, ['a', 'f', 'b', 'c', 'l', 'r'])
def test_dependency_ordering_complex_comp(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg._process_tree['a'], ['b', 'f', 'r'])
self.assertEqual(self.bsg._process_tree['b'], ['c', 'l'])
self.assertEqual(self.bsg._process_tree['c'], ['r', 'l'])
self.assertEqual(self.bsg._process_tree['r'], [])
self.assertEqual(self.bsg._process_tree['f'], [])
self.assertEqual(self.bsg._process_tree['l'], [])
def test_job_grouping(self):
self.complex_system()
self.bsg.finalize()
self.assertEqual(self.bsg.system._stages[0], 'r')
job = (self.funcs['dumb'], (None, None))
for i in self.bsg.system.stages['r'].stage:
self.assertEqual(i, job)
def test_run_without_errors_complex(self):
self.complex_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_run_without_errors_simple(self):
self.simple_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_run_without_errors_combined(self):
self.complex_system()
self.simple_system()
self.bsg.finalize()
self.assertTrue(self.bsg.system.run())
def test_ingestion_simple(self):
self.simple_system()
self.bsg.finalize()
procedural = sorted(self.bsg.specs.keys())
second = BuildSystemGenerator(self.bsg.funcs)
second.ingest(self.simple_jobs)
second.finalize()
ingested = sorted(second.specs.keys())
self.assertEqual(procedural, ingested)
def test_buildsystem_narrowing_complex(self):
self.complex_system()
self.bsg.finalize()
new_bsg = narrow_buildsystem('c', self.bsg)
for i in ['c', 'r', 'l']:
self.assertTrue(i in new_bsg._process_tree)
self.assertTrue(len(new_bsg._process_tree) == 3)
def test_initiation_funcs_initialized_tuple(self):
func = ('a', dummy_function)
bs = BuildSystemGenerator(func)
self.assertEqual(bs.funcs, {})
class TestBuildSystemGeneratorFinalize(TestCase):
@classmethod
def setUp(self):
self.bsg = BuildSystemGenerator()
def test_add_tasks_to_stage_invalid(self):
self.assertEqual(None, self.bsg._add_tasks_to_stage(False, 1, 3, {'a': 'b'}, ['a', 'b']))
def test_finalize_on_empty(self):
self.assertFalse(self.bsg._final)
self.assertEqual(None, self.bsg.system)
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_finalize_second_time(self):
self.bsg._final = True
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_finalize_with_non_clean_system(self):
self.assertFalse(self.bsg._final)
self.assertEqual(None, self.bsg.system)
self.bsg.system = True
with self.assertRaises(InvalidSystem):
self.bsg.finalize()
def test_final_toggles_final_state(self):
self.assertFalse(self.bsg._final)
self.bsg._process_job({
'stage': 'test0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b']
})
self.bsg.finalize()
self.assertTrue(self.bsg._final)
def test_final_creates_build_system_object(self):
self.assertFalse(self.bsg._final)
self.bsg._process_job({
'stage': 'test0',
'dir': '/tmp',
'cmd': 'touch',
'args': ['a', 'b']
})
self.bsg.finalize()
self.assertTrue(isinstance(self.bsg.system, BuildSystem))
