def test_f(self):
path = Path('out')
self.assertEqual(str(path), os.path.abspath('out'))
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.assertEqual(str(path), os.path.abspath('work/test/me.1234/output/lm.gz'))
def get_example_graph():
from recipe.task import test
job1 = test.Test(text=Path("input_text1.gz"))
job2 = test.Test(text=Path("input_text2.gz"))
job_merge1 = test.MergeInputs([job1.out, job2.out])
job_merge2 = test.MergeInputs([job2.out, job1.out])
job_merge3 = test.MergeInputs([job1.out, job2.out, job_merge1.out, job_merge2.out])
return SISGraph(output={'test': job_merge3.out})
def get_example_graph():
from recipe.task import test
job1 = test.Test(text=Path("input_text1.gz"))
job2 = test.Test(text=Path("input_text2.gz"))
job_merge1 = test.MergeInputs([job1.out, job2.out])
job_merge2 = test.MergeInputs([job2.out, job1.out])
job_merge3 = test.MergeInputs([job1.out, job2.out, job_merge1.out, job_merge2.out])
graph = SISGraph()
graph.add_target(OutputPath('test', job_merge3.out))
return graph
def test_jobs():
import recipe.task.test as test_recipe
job1 = test_recipe.Test(text=Path("input_text1.gz"))
job2 = test_recipe.Test(text=Path("input_text2.gz"))
job_merge1 = test_recipe.MergeInputs([job1.out, job2.out])
job_merge2 = test_recipe.MergeInputs([job2.out, job1.out])
job_merge3 = test_recipe.MergeInputs(
[job1.out, job2.out, job_merge1.out, job_merge2.out])
graph = SISGraph(output={'test': job_merge3.out})
job_engine = engine.Engine()
server = http_server.start(sis_graph=graph,
sis_engine=job_engine,
debug=True,
port=5001)
def output_path(self, filename, directory=False, cached=False):
"""
Adds output path, if directory is true a
directory will will be created automatically.
:param str filename:
:param bool directory:
:param bool cached:
:rtype: Path
"""
path = Path(filename, self, cached)
assert path.get_path() not in self._sis_outputs
self._sis_outputs[path.get_path()] = path
if directory:
self._sis_output_dirs.add(path)
return path
def __call__(self, path, *args, **kwargs):
if self.hash_overwrite and 'hash_overwrite' not in kwargs and len(args) < 3:
kwargs['hash_overwrite'] = os.path.join(self.hash_overwrite, path)
if not os.path.isabs(path):
path = os.path.join(self.origin, path)
path = os.path.relpath(path)
return Path(path, *args, **kwargs)
def test_pickle(self):
def pickle_and_check(path):
with tk.mktemp() as pickle_path:
with open(pickle_path, 'wb') as f:
pickle.dump(path, f)
with open(pickle_path, 'rb') as f:
path_unpickled = pickle.load(f)
self.assertEqual(path.__dict__, path_unpickled.__dict__)
pickle_and_check(Path('out'))
path = Path('lm.gz', MockJob('test/me.1234'))
pickle_and_check(path)
path = Path('lm.gz', MockJob('test/me.1234'), available=path_available_false)
pickle_and_check(path)
path = Variable('lm.gz', MockJob('test/me.1234'))
pickle_and_check(path)
def output_path(self, filename, directory=False, cached=False):
"""
Adds output path, if directory is true a
directory will will be created automatically.
:param str filename:
:param bool directory:
:param bool cached:
:rtype: Path
"""
path = Path(filename, self, cached)
if path.get_path() in self._sis_outputs:
logging.warning('Added output %s more than once to %s' %
(filename, self))
return self._sis_outputs[path.get_path()]
self._sis_outputs[path.get_path()] = path
if directory:
self._sis_output_dirs.add(path)
return path
def input_path(path: Union[Path, str]) -> Path:
"""Ensures a given input is a Path. Strings are automatically converted into Path objects
:param path: path that should be checked
:return: Path object
"""
if isinstance(path, str):
return Path(path)
else:
assert (isinstance(path, Path))
return path
def test_overwrite_hash(self):
path = Path('out', hash_overwrite='foo')
self.assertEqual(path._sis_hash(),
b"(Path, (tuple, (NoneType), (str, 'foo')))")
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob, hash_overwrite=('foo', 'bar'))
self.assertEqual(path._sis_hash(),
b"(Path, (tuple, (str, 'foo'), (str, 'bar')))")
def test_hash(self):
path = Path('out')
self.assertEqual(path._sis_hash(),
b"(Path, (tuple, (NoneType), (str, 'out')))")
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.assertEqual(
path._sis_hash(), b"(Path, (tuple, (MockJob, (dict, (tuple, "
b"(str, 'path'), (str, 'test/me.1234')))), "
b"(str, 'lm.gz')))")
def test_path_delay(self):
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.check_only_get_eq(path, str(path))
self.check_only_get_eq(path + '.foo', str(path) + '.foo')
self.check_only_get_eq(path[:-3], str(path)[:-3])
self.check_only_get_eq(path[-2], str(path)[-2])
self.check_only_get_eq(path[:-3] + '.foo', str(path)[:-3] + '.foo')
with tk.mktemp() as t:
var = Variable(t)
var.set(3)
self.check_only_get_eq(var + 4, 7)
self.check_only_get_eq(4 + var, 7)
self.check_only_get_eq(var * 4, 12)
self.check_only_get_eq(4 * var, 12)
def test_path_available(self):
mjob = MockJob('test/me.1234')
path = Path('lm.gz', mjob)
self.assertEqual(path.available(), False)
mjob._sis_finished = lambda: True
mjob.path_available = lambda path: True
self.assertEqual(path.available(), True)
with tk.mktemp() as test_path:
path = Path(test_path)
self.assertEqual(path.available(), False)
with open(test_path, 'wb') as _:
pass
self.assertEqual(path.available(), True)
path = Path('lm.gz', mjob, available=path_available_false)
self.assertEqual(path.available(), False)
path = Path('lm.gz', mjob, available=path_available_true)
self.assertEqual(path.available(), True)
def test_run(self):
with execute_in_dir(TEST_DIR):
from recipe.task.test import Test
job = Test(text=Path("input_text.gz"))
job._sis_setup_directory()
def __call__(self, path, *args, **kwargs):
if not os.path.isabs(path):
path = os.path.join(self.origin, path)
path = os.path.relpath(path)
return Path(path, *args, **kwargs)
t = HttpThread(port, debug=debug)
if thread:
t.daemon = True
t.start()
else:
t.run()
return t
if __name__ == '__main__':
from recipe import get_recipe
from job_path import Path
from graph import SISGraph
test_recipe = get_recipe('task/test')
job1 = test_recipe.Test(text=Path("input_text1.gz"))
job2 = test_recipe.Test(text=Path("input_text2.gz"))
job_merge1 = test_recipe.MergeInputs([job1.out, job2.out])
job_merge2 = test_recipe.MergeInputs([job2.out, job1.out])
job_merge3 = test_recipe.MergeInputs(
[job1.out, job2.out, job_merge1.out, job_merge2.out])
sis_graph = SISGraph(output={'test': job_merge3.out})
import engine
sis_engine = engine.Engine()
port = int(sys.argv[1])
try:
server = start(sis_graph=sis_graph,
sis_engine=sis_engine,
debug=True,
port=port,