예제 #1
0
    def test_f(self):
        path = Path('out')
        self.assertEqual(str(path), os.path.abspath('out'))

        mjob = MockJob('test/me.1234')
        path = Path('lm.gz', mjob)
        self.assertEqual(str(path), os.path.abspath('work/test/me.1234/output/lm.gz'))
예제 #2
0
def get_example_graph():
    from recipe.task import test
    job1 = test.Test(text=Path("input_text1.gz"))
    job2 = test.Test(text=Path("input_text2.gz"))
    job_merge1 = test.MergeInputs([job1.out, job2.out])
    job_merge2 = test.MergeInputs([job2.out, job1.out])
    job_merge3 = test.MergeInputs([job1.out, job2.out, job_merge1.out, job_merge2.out])
    return SISGraph(output={'test': job_merge3.out})
예제 #3
0
def get_example_graph():
    from recipe.task import test
    job1 = test.Test(text=Path("input_text1.gz"))
    job2 = test.Test(text=Path("input_text2.gz"))
    job_merge1 = test.MergeInputs([job1.out, job2.out])
    job_merge2 = test.MergeInputs([job2.out, job1.out])
    job_merge3 = test.MergeInputs([job1.out, job2.out, job_merge1.out, job_merge2.out])
    graph = SISGraph()
    graph.add_target(OutputPath('test', job_merge3.out))
    return graph
예제 #4
0
def test_jobs():
    import recipe.task.test as test_recipe
    job1 = test_recipe.Test(text=Path("input_text1.gz"))
    job2 = test_recipe.Test(text=Path("input_text2.gz"))
    job_merge1 = test_recipe.MergeInputs([job1.out, job2.out])
    job_merge2 = test_recipe.MergeInputs([job2.out, job1.out])
    job_merge3 = test_recipe.MergeInputs(
        [job1.out, job2.out, job_merge1.out, job_merge2.out])
    graph = SISGraph(output={'test': job_merge3.out})
    job_engine = engine.Engine()

    server = http_server.start(sis_graph=graph,
                               sis_engine=job_engine,
                               debug=True,
                               port=5001)
예제 #5
0
    def output_path(self, filename, directory=False, cached=False):
        """
        Adds output path, if directory is true a
        directory will will be created automatically.

        :param str filename:
        :param bool directory:
        :param bool cached:
        :rtype: Path
        """
        path = Path(filename, self, cached)
        assert path.get_path() not in self._sis_outputs
        self._sis_outputs[path.get_path()] = path
        if directory:
            self._sis_output_dirs.add(path)
        return path
예제 #6
0
 def __call__(self, path, *args, **kwargs):
     if self.hash_overwrite and 'hash_overwrite' not in kwargs and len(args) < 3:
         kwargs['hash_overwrite'] = os.path.join(self.hash_overwrite, path)
     if not os.path.isabs(path):
         path = os.path.join(self.origin, path)
         path = os.path.relpath(path)
     return Path(path, *args, **kwargs)
예제 #7
0
    def test_pickle(self):
        def pickle_and_check(path):
            with tk.mktemp() as pickle_path:
                with open(pickle_path, 'wb') as f:
                    pickle.dump(path, f)
                with open(pickle_path, 'rb') as f:
                    path_unpickled = pickle.load(f)
            self.assertEqual(path.__dict__, path_unpickled.__dict__)

        pickle_and_check(Path('out'))

        path = Path('lm.gz', MockJob('test/me.1234'))
        pickle_and_check(path)

        path = Path('lm.gz', MockJob('test/me.1234'), available=path_available_false)
        pickle_and_check(path)

        path = Variable('lm.gz', MockJob('test/me.1234'))
        pickle_and_check(path)
예제 #8
0
파일: job.py 프로젝트: mmz33/sisyphus
    def output_path(self, filename, directory=False, cached=False):
        """
        Adds output path, if directory is true a
        directory will will be created automatically.

        :param str filename:
        :param bool directory:
        :param bool cached:
        :rtype: Path
        """
        path = Path(filename, self, cached)
        if path.get_path() in self._sis_outputs:
            logging.warning('Added output %s more than once to %s' %
                            (filename, self))
            return self._sis_outputs[path.get_path()]
        self._sis_outputs[path.get_path()] = path
        if directory:
            self._sis_output_dirs.add(path)
        return path
예제 #9
0
def input_path(path: Union[Path, str]) -> Path:
    """Ensures a given input is a Path. Strings are automatically converted into Path objects

    :param path: path that should be checked
    :return: Path object
    """
    if isinstance(path, str):
        return Path(path)
    else:
        assert (isinstance(path, Path))
        return path
예제 #10
0
    def test_overwrite_hash(self):
        path = Path('out', hash_overwrite='foo')
        self.assertEqual(path._sis_hash(),
                         b"(Path, (tuple, (NoneType), (str, 'foo')))")

        mjob = MockJob('test/me.1234')
        path = Path('lm.gz', mjob, hash_overwrite=('foo', 'bar'))
        self.assertEqual(path._sis_hash(),
                         b"(Path, (tuple, (str, 'foo'), (str, 'bar')))")
예제 #11
0
    def test_hash(self):
        path = Path('out')
        self.assertEqual(path._sis_hash(),
                         b"(Path, (tuple, (NoneType), (str, 'out')))")

        mjob = MockJob('test/me.1234')
        path = Path('lm.gz', mjob)
        self.assertEqual(
            path._sis_hash(), b"(Path, (tuple, (MockJob, (dict, (tuple, "
            b"(str, 'path'), (str, 'test/me.1234')))), "
            b"(str, 'lm.gz')))")
예제 #12
0
    def test_path_delay(self):
        mjob = MockJob('test/me.1234')
        path = Path('lm.gz', mjob)

        self.check_only_get_eq(path, str(path))
        self.check_only_get_eq(path + '.foo', str(path) + '.foo')
        self.check_only_get_eq(path[:-3], str(path)[:-3])
        self.check_only_get_eq(path[-2], str(path)[-2])
        self.check_only_get_eq(path[:-3] + '.foo', str(path)[:-3] + '.foo')

        with tk.mktemp() as t:
            var = Variable(t)
            var.set(3)
            self.check_only_get_eq(var + 4, 7)
            self.check_only_get_eq(4 + var, 7)
            self.check_only_get_eq(var * 4, 12)
            self.check_only_get_eq(4 * var, 12)
예제 #13
0
    def test_path_available(self):
        mjob = MockJob('test/me.1234')
        path = Path('lm.gz', mjob)
        self.assertEqual(path.available(), False)

        mjob._sis_finished = lambda: True
        mjob.path_available = lambda path: True
        self.assertEqual(path.available(), True)

        with tk.mktemp() as test_path:
            path = Path(test_path)
            self.assertEqual(path.available(), False)
            with open(test_path, 'wb') as _:
                pass
            self.assertEqual(path.available(), True)

        path = Path('lm.gz', mjob, available=path_available_false)
        self.assertEqual(path.available(), False)
        path = Path('lm.gz', mjob, available=path_available_true)
        self.assertEqual(path.available(), True)
예제 #14
0
 def test_run(self):
     with execute_in_dir(TEST_DIR):
         from recipe.task.test import Test
         job = Test(text=Path("input_text.gz"))
         job._sis_setup_directory()
예제 #15
0
파일: toolkit.py 프로젝트: ArneNx/sisyphus
 def __call__(self, path, *args, **kwargs):
     if not os.path.isabs(path):
         path = os.path.join(self.origin, path)
         path = os.path.relpath(path)
     return Path(path, *args, **kwargs)
예제 #16
0
    t = HttpThread(port, debug=debug)
    if thread:
        t.daemon = True
        t.start()
    else:
        t.run()
    return t


if __name__ == '__main__':
    from recipe import get_recipe
    from job_path import Path
    from graph import SISGraph

    test_recipe = get_recipe('task/test')
    job1 = test_recipe.Test(text=Path("input_text1.gz"))
    job2 = test_recipe.Test(text=Path("input_text2.gz"))
    job_merge1 = test_recipe.MergeInputs([job1.out, job2.out])
    job_merge2 = test_recipe.MergeInputs([job2.out, job1.out])
    job_merge3 = test_recipe.MergeInputs(
        [job1.out, job2.out, job_merge1.out, job_merge2.out])
    sis_graph = SISGraph(output={'test': job_merge3.out})
    import engine
    sis_engine = engine.Engine()

    port = int(sys.argv[1])
    try:
        server = start(sis_graph=sis_graph,
                       sis_engine=sis_engine,
                       debug=True,
                       port=port,