Esempio n. 1
0
    def emit_task(self,
                  abstraction,
                  function,
                  command,
                  inputs,
                  outputs,
                  options,
                  symbol=None):
        """ Write task to DAG file. """
        # Track inputs and outputs.
        if self.track_imports:
            for i in inputs:
                self.inputs.add(i)

        if self.track_exports:
            for o in outputs:
                self.outputs.add(o)

        debug(
            D_ENGINE, 'Emitting {0}, [{1}], [{2}], {3}'.format(
                command, ', '.join(map(str, inputs)),
                ', '.join(map(str, outputs)), options))

        # Write task outputs and inputs
        self.dag_file.write('{0}: {1}\n'.format(' '.join(map(str, outputs)),
                                                ' '.join(map(str, inputs))))

        # Write debugging symbols if enabled
        if CurrentScript().include_symbols:
            if abstraction == SENTINEL:
                self.dag_file.write(
                    '\t'.join(['', '# SYMBOL', str(function)]) + '\n')
            else:
                self.dag_file.write('\t'.join(
                    ['', '# SYMBOL', str(abstraction)]) + '\n')

        # if a symbol is provided
        if symbol:
            self.dag_file.write('@SYMBOL="' + symbol + '"\n')

        # Write environmental variables
        if options.local:
            self.dag_file.write('@BATCH_LOCAL=1\n')
        if options.batch:
            self.dag_file.write('@BATCH_OPTIONS={0}\n'.format(options.batch))
        if options.collect:
            self.dag_file.write('@_MAKEFLOW_COLLECT_LIST+={0}\n'.format(
                ' '.join(map(str, options.collect))))
        for k, v in list(options.environment.items()):
            self.dag_file.write('@{0}={1}\n'.format(k, v))

        # Write task command
        self.dag_file.write('\t{0}\n'.format(command))
        self.dag_file.flush()
Esempio n. 2
0
    def schedule(self, abstraction, function, command, inputs, outputs,
                 options, symbol=None):
        """ Schedule task for execution. """
        debug(D_NEST,
            'Scheduling task({0}, {1}, {2}, [{3}], [{4}], {5}) for {6}'.format(
            abstraction, function, command,
            ', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
            options, self))

        if abstraction is None:
            abstraction = SENTINEL
        self.tasks.append(
            [abstraction, function, command, inputs, outputs, options, symbol])
Esempio n. 3
0
    def schedule(self, abstraction, function, command, inputs, outputs,
        options):
        """ Schedule task for execution. """
        debug(D_NEST,
            'Scheduling task({0}, {1}, {2}, [{3}], [{4}], {5}) for {6}'.format(
            abstraction, function, command,
            ', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
            options, self))

        if abstraction is None:
            abstraction = SENTINEL
        self.tasks.append(
            [abstraction, function, command, inputs, outputs, options])
Esempio n. 4
0
def Or(*filters):
    if not filters: return ''

    if isinstance(filters[0], str):
        return '(' + ' OR '.join(filters) + ')'
    else:
        return lambda filters: any(map(lambda d: f(d), filters))
Esempio n. 5
0
    def __call__(self, *args, **kwds):
        self.cmd_format = 'MAKEFLOW "{0}" "{1}" "{2}"'.format(
            self.dag_path, self.work_dir, self.wrapper)

        # Prune inputs of outputs (happens when we have a SubNest).
        self.inputs = [i for i in self.inputs if str(i) not in set(map(str, self.outputs))]
        Engine.__call__(self, *args, **kwds)
Esempio n. 6
0
def Or(*filters):
    if not filters: return ''

    if isinstance(filters[0], str):
        return '(' + ' OR '.join(filters) + ')'
    else:
        return lambda filters: any(map(lambda d: f(d), filters))
Esempio n. 7
0
def And(*filters):
    if not filters: return ''

    if isinstance(filters[0], str):
        return '(' + ' AND '.join(filters) + ')'
    else:
        return lambda filters: all(map(lambda d: f(d), filters))
Esempio n. 8
0
def And(*filters):
    if not filters: return ''

    if isinstance(filters[0], str):
        return '(' + ' AND '.join(filters) + ')'
    else:
        return lambda filters: all(map(lambda d: f(d), filters))
Esempio n. 9
0
    def __call__(self, *args, **kwds):
        self.cmd_format = 'MAKEFLOW "{0}" "{1}" "{2}"'.format(
            self.dag_path, self.work_dir, self.wrapper)

        # Prune inputs of outputs (happens when we have a SubNest).
        self.inputs = [
            i for i in self.inputs if str(i) not in set(map(str, self.outputs))
        ]
        Engine.__call__(self, *args, **kwds)
Esempio n. 10
0
    def emit_task(self, abstraction, function, command, inputs, outputs, options, symbol=None):
        """ Write task to DAG file. """
        # Track inputs and outputs.
        if self.track_imports:
            for i in inputs:
                self.inputs.add(i)

        if self.track_exports:
            for o in outputs:
                self.outputs.add(o)

        debug(D_ENGINE, 'Emitting {0}, [{1}], [{2}], {3}'.format(
            command, ', '.join(map(str, inputs)), ', '.join(map(str, outputs)),
            options))

        # Write task outputs and inputs
        self.dag_file.write('{0}: {1}\n'.format(
            ' '.join(map(str, outputs)), ' '.join(map(str, inputs))))

        # Write debugging symbols if enabled
        if CurrentScript().include_symbols:
            if abstraction == SENTINEL:
                self.dag_file.write('\t'.join(['', '# SYMBOL', str(function)]) + '\n')
            else:
                self.dag_file.write('\t'.join(['', '# SYMBOL', str(abstraction)]) + '\n')

        # if a symbol is provided
        if symbol:
            self.dag_file.write('@SYMBOL="' + symbol+'"\n')

        # Write environmental variables
        if options.local:
            self.dag_file.write('@BATCH_LOCAL=1\n')
        if options.batch:
            self.dag_file.write('@BATCH_OPTIONS={0}\n'.format(options.batch))
        if options.collect:
            self.dag_file.write('@_MAKEFLOW_COLLECT_LIST+={0}\n'.format(
                ' '.join(map(str, options.collect))))
        for k, v in list(options.environment.items()):
            self.dag_file.write('@{0}={1}\n'.format(k, v))

        # Write task command
        self.dag_file.write('\t{0}\n'.format(command))
        self.dag_file.flush()
Esempio n. 11
0
    def test_00_string(self):
        self.assertEqual(parse_string_list('a'), ['a'])

    def test_01_list(self):
        l = ['a']
        self.assertEqual(parse_string_list(l), l)

    def test_02_tuple(self):
        t = ('a', )
        self.assertEqual(parse_string_list(t), ['a'])

    def test_03_generator(self):
        g = list(range(10))
        self.assertEqual(parse_string_list(g), list(map(str, g)))


# Main execution ---------------------------------------------------------------

TestCases = [
    FindExecutableTestCase,
    NormalizePathTestCase,
    ParseStringListTestCase,
]

if __name__ == '__main__':
    test_runner = TextTestRunner(verbosity=2)
    test_suite = TestSuite(
        list(map(TestLoader().loadTestsFromTestCase, TestCases)))
    test_runner.run(test_suite)

# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
Esempio n. 12
0
 def __or__(self, other):
     debug(D_DATASET, 'or: {0} {1}'.format(self._field, other))
     return Or(*map(lambda o: self == o, other))
Esempio n. 13
0
    def test_00_string(self):
        self.assertEqual(parse_string_list('a'), ['a'])

    def test_01_list(self):
        l = ['a']
        self.assertEqual(parse_string_list(l), l)

    def test_02_tuple(self):
        t = ('a', )
        self.assertEqual(parse_string_list(t), ['a'])

    def test_03_generator(self):
        g = range(10)
        self.assertEqual(parse_string_list(g), list(map(str, g)))


# Main execution ---------------------------------------------------------------

TestCases = [
    FindExecutableTestCase,
    NormalizePathTestCase,
    ParseStringListTestCase,
]

if __name__ == '__main__':
    test_runner = TextTestRunner(verbosity=2)
    test_suite = TestSuite(map(TestLoader().loadTestsFromTestCase, TestCases))
    test_runner.run(test_suite)

# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
Esempio n. 14
0
 def test_03_generator(self):
     g = range(10)
     self.assertEqual(parse_string_list(g), list(map(str, g)))
Esempio n. 15
0
# Parse string list test case --------------------------------------------------


class ParseStringListTestCase(TestCase):
    def test_00_string(self):
        self.assertEqual(parse_string_list("a"), ["a"])

    def test_01_list(self):
        l = ["a"]
        self.assertEqual(parse_string_list(l), l)

    def test_02_tuple(self):
        t = ("a",)
        self.assertEqual(parse_string_list(t), ["a"])

    def test_03_generator(self):
        g = range(10)
        self.assertEqual(parse_string_list(g), list(map(str, g)))


# Main execution ---------------------------------------------------------------

TestCases = [FindExecutableTestCase, NormalizePathTestCase, ParseStringListTestCase]

if __name__ == "__main__":
    test_runner = TextTestRunner(verbosity=2)
    test_suite = TestSuite(map(TestLoader().loadTestsFromTestCase, TestCases))
    test_runner.run(test_suite)

# vim: set sts=4 sw=4 ts=8 expandtab ft=python: --------------------------------
Esempio n. 16
0
 def test_03_generator(self):
     g = range(10)
     self.assertEqual(parse_string_list(g), list(map(str, g)))
Esempio n. 17
0
 def __and__(self, other):
     debug(D_DATASET, 'and: {0} {1}'.format(self._field, other))
     return And(*map(lambda o: self == o, other))
Esempio n. 18
0
 def __and__(self, other):
     debug(D_DATASET, 'and: {0} {1}'.format(self._field, other))
     return And(*map(lambda o: self == o, other))
Esempio n. 19
0
 def __or__(self, other):
     debug(D_DATASET, 'or: {0} {1}'.format(self._field, other))
     return Or(*map(lambda o: self == o, other))