def test_create_data_prepare_with_param_func_and_large_parameter(self):
do_test1 = False
a_value = None
large_param = 'b' * 251
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
project = create_data_prepare({
'Test2': _test_global,
'Test1': test1
}, {
'name': 'test',
'a':
"{{ search_preprocess_directory('tests.test_projects_data._test_global', {'name': 'test', 'b': 'b'*251}).joinpath('aaa').open('r').read() }}",
'b': large_param
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(project.requires()[0].output().exists())
ok_(do_test1)
eq_(a_value, 'aaa')
def test_create_data_prepare_has_two_first_is_done_and_update(self):
do_test1 = False
a_value = None
do_test2 = False
b_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
def test2(b, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test2
nonlocal b_value
do_test2 = True
b_value = b
project = create_data_prepare({'Test2': test2}, {'a': 1, 'b': 2})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(do_test2)
eq_(b_value, 2)
do_test2 = False
project = create_data_prepare({
'Test2': test2,
'Test1': test1
}, {
'a': 1,
'b': 3
},
update_task='Test2')
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(project.requires()[0].output().exists())
project.requires()[0].output().remove()
ok_(do_test1)
eq_(a_value, 1)
ok_(do_test2)
eq_(b_value, 3)
def test_run_once(self):
_save_conf({'epochs': 1}, {}, {})
project = RunOnceProject(**{
'runner': 'image_recognition_trainer',
'param_path': CONF_PATH,
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(project.artifact_directory.joinpath('model').exists())
def test_run_once_with_before_preprocess_class_method(self):
_save_conf({
'epochs': 1,
}, {
'batch_size':
2,
'adjusted_shape': (128, 128),
'train_image_directory':
'{{ __import__("pathlib").Path("%s") }}' %
pathlib.Path(__file__).parent.joinpath('data/openimages/train'),
'test_image_directory':
'{{ __import__("pathlib").Path("%s") }}' % pathlib.Path(
__file__).parent.joinpath('data/openimages/validation'),
'train_data_directory':
'{{ search_preprocess_directory("dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord", preprocess_params).joinpath("train") }}',
'test_data_directory':
'{{ search_preprocess_directory("dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord", preprocess_params).joinpath("test") }}',
}, {
'projects':
OrderedDict(
**{
'Convert':
'dataset.open_images.OpenImagesClassificationDataset.convert_tfrecord',
}),
'parameters': {
'before_artifact_directory':
'{{ __import__("pathlib").Path("%s") }}' %
pathlib.Path(__file__).parent.joinpath('data/openimages'),
'split_num':
1,
'name':
'openimages'
}
})
project = RunOnceProject(
**{
'runner': 'image_recognition_trainer',
'dataset': 'openimages',
'param_path': CONF_PATH,
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(project.artifact_directory.joinpath('model').exists())
def test_create_data_prepare(self):
do_test1 = False
a_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
project = create_data_prepare({'Test1': test1}, {'a': 1, 'b': 2})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(do_test1)
eq_(a_value, 1)
def test_create_data_prepare_save_and_load(self):
do_test1 = False
a_value = None
before_value = None
do_test2 = False
b_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
nonlocal before_value
do_test1 = True
a_value = a
with before_artifact_directory.joinpath('aaa').open('r') as f:
before_value = f.read()
def test2(b, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test2
nonlocal b_value
do_test2 = True
b_value = b
with artifact_directory.joinpath('aaa').open('w') as f:
f.write('aaa')
project = create_data_prepare({
'Test2': test2,
'Test1': test1
}, {
'a': 1,
'b': 2
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(project.requires()[0].output().exists())
project.requires()[0].output().remove()
ok_(do_test1)
eq_(a_value, 1)
ok_(do_test2)
eq_(b_value, 2)
eq_(before_value, 'aaa')
def test_run_once_add_preprocess(self):
_save_conf(
{'epochs': 1}, {'data_path': '{{ before_artifact_directory }}'}, {
'projects':
OrderedDict(
**{
'Download': 'dataset.mnist_from_raw.download_data',
'Decompose': 'dataset.mnist_from_raw.decompose_data',
})
})
project = RunOnceProject(
**{
'runner': 'image_recognition_trainer',
'dataset': 'mnistraw',
'param_path': CONF_PATH,
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(project.artifact_directory.joinpath('model').exists())