def test_create_data_prepare_with_param_func_and_large_parameter(self): do_test1 = False a_value = None large_param = 'b' * 251 def test1(a, artifact_directory=None, before_artifact_directory=None): nonlocal do_test1 nonlocal a_value do_test1 = True a_value = a project = create_data_prepare({ 'Test2': _test_global, 'Test1': test1 }, { 'name': 'test', 'a': "{{ search_preprocess_directory('tests.test_projects_data._test_global', {'name': 'test', 'b': 'b'*251}).joinpath('aaa').open('r').read() }}", 'b': large_param }) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) ok_(project.requires()[0].output().exists()) ok_(do_test1) eq_(a_value, 'aaa')
def test_create_data_prepare_has_two_first_is_done_and_update(self): do_test1 = False a_value = None do_test2 = False b_value = None def test1(a, artifact_directory=None, before_artifact_directory=None): nonlocal do_test1 nonlocal a_value do_test1 = True a_value = a def test2(b, artifact_directory=None, before_artifact_directory=None): nonlocal do_test2 nonlocal b_value do_test2 = True b_value = b project = create_data_prepare({'Test2': test2}, {'a': 1, 'b': 2}) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) ok_(do_test2) eq_(b_value, 2) do_test2 = False project = create_data_prepare({ 'Test2': test2, 'Test1': test1 }, { 'a': 1, 'b': 3 }, update_task='Test2') run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) project.output().remove() ok_(project.requires()[0].output().exists()) project.requires()[0].output().remove() ok_(do_test1) eq_(a_value, 1) ok_(do_test2) eq_(b_value, 3)
def test_run_once(self): _save_conf({'epochs': 1}, {}, {}) project = RunOnceProject(**{ 'runner': 'image_recognition_trainer', 'param_path': CONF_PATH, }) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) ok_(project.artifact_directory.joinpath('model').exists())
def test_run_once_with_before_preprocess_class_method(self): _save_conf({ 'epochs': 1, }, { 'batch_size': 2, 'adjusted_shape': (128, 128), 'train_image_directory': '{{ __import__("pathlib").Path("%s") }}' % pathlib.Path(__file__).parent.joinpath('data/openimages/train'), 'test_image_directory': '{{ __import__("pathlib").Path("%s") }}' % pathlib.Path( __file__).parent.joinpath('data/openimages/validation'), 'train_data_directory': '{{ search_preprocess_directory("dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord", preprocess_params).joinpath("train") }}', 'test_data_directory': '{{ search_preprocess_directory("dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord", preprocess_params).joinpath("test") }}', }, { 'projects': OrderedDict( **{ 'Convert': 'dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord', }), 'parameters': { 'before_artifact_directory': '{{ __import__("pathlib").Path("%s") }}' % pathlib.Path(__file__).parent.joinpath('data/openimages'), 'split_num': 1, 'name': 'openimages' } }) project = RunOnceProject( **{ 'runner': 'image_recognition_trainer', 'dataset': 'openimages', 'param_path': CONF_PATH, }) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) ok_(project.artifact_directory.joinpath('model').exists())
def test_create_data_prepare(self): do_test1 = False a_value = None def test1(a, artifact_directory=None, before_artifact_directory=None): nonlocal do_test1 nonlocal a_value do_test1 = True a_value = a project = create_data_prepare({'Test1': test1}, {'a': 1, 'b': 2}) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) project.output().remove() ok_(do_test1) eq_(a_value, 1)
def test_create_data_prepare_save_and_load(self): do_test1 = False a_value = None before_value = None do_test2 = False b_value = None def test1(a, artifact_directory=None, before_artifact_directory=None): nonlocal do_test1 nonlocal a_value nonlocal before_value do_test1 = True a_value = a with before_artifact_directory.joinpath('aaa').open('r') as f: before_value = f.read() def test2(b, artifact_directory=None, before_artifact_directory=None): nonlocal do_test2 nonlocal b_value do_test2 = True b_value = b with artifact_directory.joinpath('aaa').open('w') as f: f.write('aaa') project = create_data_prepare({ 'Test2': test2, 'Test1': test1 }, { 'a': 1, 'b': 2 }) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) project.output().remove() ok_(project.requires()[0].output().exists()) project.requires()[0].output().remove() ok_(do_test1) eq_(a_value, 1) ok_(do_test2) eq_(b_value, 2) eq_(before_value, 'aaa')
def test_run_once_add_preprocess(self): _save_conf( {'epochs': 1}, {'data_path': '{{ before_artifact_directory }}'}, { 'projects': OrderedDict( **{ 'Download': 'dataset.mnist_from_raw.download_data', 'Decompose': 'dataset.mnist_from_raw.decompose_data', }) }) project = RunOnceProject( **{ 'runner': 'image_recognition_trainer', 'dataset': 'mnistraw', 'param_path': CONF_PATH, }) run_result = luigi.build([project], worker_scheduler_factory=DummyFactory()) ok_(run_result) ok_(project.output().exists()) ok_(project.artifact_directory.joinpath('model').exists())