def emit_task(self,
abstraction,
function,
command,
inputs,
outputs,
options,
symbol=None):
""" Write task to DAG file. """
# Track inputs and outputs.
if self.track_imports:
for i in inputs:
self.inputs.add(i)
if self.track_exports:
for o in outputs:
self.outputs.add(o)
debug(
D_ENGINE, 'Emitting {0}, [{1}], [{2}], {3}'.format(
command, ', '.join(map(str, inputs)),
', '.join(map(str, outputs)), options))
# Write task outputs and inputs
self.dag_file.write('{0}: {1}\n'.format(' '.join(map(str, outputs)),
' '.join(map(str, inputs))))
# Write debugging symbols if enabled
if CurrentScript().include_symbols:
if abstraction == SENTINEL:
self.dag_file.write(
'\t'.join(['', '# SYMBOL', str(function)]) + '\n')
else:
self.dag_file.write('\t'.join(
['', '# SYMBOL', str(abstraction)]) + '\n')
# if a symbol is provided
if symbol:
self.dag_file.write('@SYMBOL="' + symbol + '"\n')
# Write environmental variables
if options.local:
self.dag_file.write('@BATCH_LOCAL=1\n')
if options.batch:
self.dag_file.write('@BATCH_OPTIONS={0}\n'.format(options.batch))
if options.collect:
self.dag_file.write('@_MAKEFLOW_COLLECT_LIST+={0}\n'.format(
' '.join(map(str, options.collect))))
for k, v in list(options.environment.items()):
self.dag_file.write('@{0}={1}\n'.format(k, v))
# Write task command
self.dag_file.write('\t{0}\n'.format(command))
self.dag_file.flush()
def schedule(self, abstraction, function, command, inputs, outputs,
options, symbol=None):
""" Schedule task for execution. """
debug(D_NEST,
'Scheduling task({0}, {1}, {2}, [{3}], [{4}], {5}) for {6}'.format(
abstraction, function, command,
', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
options, self))
if abstraction is None:
abstraction = SENTINEL
self.tasks.append(
[abstraction, function, command, inputs, outputs, options, symbol])
def schedule(self, abstraction, function, command, inputs, outputs,
options):
""" Schedule task for execution. """
debug(D_NEST,
'Scheduling task({0}, {1}, {2}, [{3}], [{4}], {5}) for {6}'.format(
abstraction, function, command,
', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
options, self))
if abstraction is None:
abstraction = SENTINEL
self.tasks.append(
[abstraction, function, command, inputs, outputs, options])
def Or(*filters):
if not filters: return ''
if isinstance(filters[0], str):
return '(' + ' OR '.join(filters) + ')'
else:
return lambda filters: any(map(lambda d: f(d), filters))
def __call__(self, *args, **kwds):
self.cmd_format = 'MAKEFLOW "{0}" "{1}" "{2}"'.format(
self.dag_path, self.work_dir, self.wrapper)
# Prune inputs of outputs (happens when we have a SubNest).
self.inputs = [i for i in self.inputs if str(i) not in set(map(str, self.outputs))]
Engine.__call__(self, *args, **kwds)
def Or(*filters):
if not filters: return ''
if isinstance(filters[0], str):
return '(' + ' OR '.join(filters) + ')'
else:
return lambda filters: any(map(lambda d: f(d), filters))
def And(*filters):
if not filters: return ''
if isinstance(filters[0], str):
return '(' + ' AND '.join(filters) + ')'
else:
return lambda filters: all(map(lambda d: f(d), filters))
def And(*filters):
if not filters: return ''
if isinstance(filters[0], str):
return '(' + ' AND '.join(filters) + ')'
else:
return lambda filters: all(map(lambda d: f(d), filters))
def __call__(self, *args, **kwds):
self.cmd_format = 'MAKEFLOW "{0}" "{1}" "{2}"'.format(
self.dag_path, self.work_dir, self.wrapper)
# Prune inputs of outputs (happens when we have a SubNest).
self.inputs = [
i for i in self.inputs if str(i) not in set(map(str, self.outputs))
]
Engine.__call__(self, *args, **kwds)
def emit_task(self, abstraction, function, command, inputs, outputs, options, symbol=None):
""" Write task to DAG file. """
# Track inputs and outputs.
if self.track_imports:
for i in inputs:
self.inputs.add(i)
if self.track_exports:
for o in outputs:
self.outputs.add(o)
debug(D_ENGINE, 'Emitting {0}, [{1}], [{2}], {3}'.format(
command, ', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
options))
# Write task outputs and inputs
self.dag_file.write('{0}: {1}\n'.format(
' '.join(map(str, outputs)), ' '.join(map(str, inputs))))
# Write debugging symbols if enabled
if CurrentScript().include_symbols:
if abstraction == SENTINEL:
self.dag_file.write('\t'.join(['', '# SYMBOL', str(function)]) + '\n')
else:
self.dag_file.write('\t'.join(['', '# SYMBOL', str(abstraction)]) + '\n')
# if a symbol is provided
if symbol:
self.dag_file.write('@SYMBOL="' + symbol+'"\n')
# Write environmental variables
if options.local:
self.dag_file.write('@BATCH_LOCAL=1\n')
if options.batch:
self.dag_file.write('@BATCH_OPTIONS={0}\n'.format(options.batch))
if options.collect:
self.dag_file.write('@_MAKEFLOW_COLLECT_LIST+={0}\n'.format(
' '.join(map(str, options.collect))))
for k, v in list(options.environment.items()):
self.dag_file.write('@{0}={1}\n'.format(k, v))
# Write task command
self.dag_file.write('\t{0}\n'.format(command))
self.dag_file.flush()
def test_00_string(self):
self.assertEqual(parse_string_list('a'), ['a'])
def test_01_list(self):
l = ['a']
self.assertEqual(parse_string_list(l), l)
def test_02_tuple(self):
t = ('a', )
self.assertEqual(parse_string_list(t), ['a'])
def test_03_generator(self):
g = list(range(10))
self.assertEqual(parse_string_list(g), list(map(str, g)))
# Main execution ---------------------------------------------------------------
TestCases = [
FindExecutableTestCase,
NormalizePathTestCase,
ParseStringListTestCase,
]
if __name__ == '__main__':
test_runner = TextTestRunner(verbosity=2)
test_suite = TestSuite(
list(map(TestLoader().loadTestsFromTestCase, TestCases)))
test_runner.run(test_suite)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
def __or__(self, other):
debug(D_DATASET, 'or: {0} {1}'.format(self._field, other))
return Or(*map(lambda o: self == o, other))
def test_00_string(self):
self.assertEqual(parse_string_list('a'), ['a'])
def test_01_list(self):
l = ['a']
self.assertEqual(parse_string_list(l), l)
def test_02_tuple(self):
t = ('a', )
self.assertEqual(parse_string_list(t), ['a'])
def test_03_generator(self):
g = range(10)
self.assertEqual(parse_string_list(g), list(map(str, g)))
# Main execution ---------------------------------------------------------------
TestCases = [
FindExecutableTestCase,
NormalizePathTestCase,
ParseStringListTestCase,
]
if __name__ == '__main__':
test_runner = TextTestRunner(verbosity=2)
test_suite = TestSuite(map(TestLoader().loadTestsFromTestCase, TestCases))
test_runner.run(test_suite)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
def test_03_generator(self):
g = range(10)
self.assertEqual(parse_string_list(g), list(map(str, g)))
# Parse string list test case --------------------------------------------------
class ParseStringListTestCase(TestCase):
def test_00_string(self):
self.assertEqual(parse_string_list("a"), ["a"])
def test_01_list(self):
l = ["a"]
self.assertEqual(parse_string_list(l), l)
def test_02_tuple(self):
t = ("a",)
self.assertEqual(parse_string_list(t), ["a"])
def test_03_generator(self):
g = range(10)
self.assertEqual(parse_string_list(g), list(map(str, g)))
# Main execution ---------------------------------------------------------------
TestCases = [FindExecutableTestCase, NormalizePathTestCase, ParseStringListTestCase]
if __name__ == "__main__":
test_runner = TextTestRunner(verbosity=2)
test_suite = TestSuite(map(TestLoader().loadTestsFromTestCase, TestCases))
test_runner.run(test_suite)
# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
def test_03_generator(self):
g = range(10)
self.assertEqual(parse_string_list(g), list(map(str, g)))
def __and__(self, other):
debug(D_DATASET, 'and: {0} {1}'.format(self._field, other))
return And(*map(lambda o: self == o, other))
def __and__(self, other):
debug(D_DATASET, 'and: {0} {1}'.format(self._field, other))
return And(*map(lambda o: self == o, other))
def __or__(self, other):
debug(D_DATASET, 'or: {0} {1}'.format(self._field, other))
return Or(*map(lambda o: self == o, other))