def test_create_data_prepare_with_param_func_list(self):
do_test1 = False
a_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
project = create_data_prepare({
'Test2': _test_global,
'Test1': test1
}, {
'name': 'test',
'a': {
'aa': [
"{{ search_preprocess_directory('tests.test_projects_data._test_global', {'name': 'test', 'b': [{'bb': 'gggg'}]}).joinpath('aaa').open('r').read() }}"
]
},
'b': [{
'bb': "gggg"
}]
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(project.requires()[0].output().exists())
ok_(do_test1)
eq_(a_value, {'aa': ['aaa']})
def test_create_data_prepare_has_two_first_is_done_and_update(self):
do_test1 = False
a_value = None
do_test2 = False
b_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
def test2(b, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test2
nonlocal b_value
do_test2 = True
b_value = b
project = create_data_prepare({'Test2': test2}, {'a': 1, 'b': 2})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
ok_(do_test2)
eq_(b_value, 2)
do_test2 = False
project = create_data_prepare({
'Test2': test2,
'Test1': test1
}, {
'a': 1,
'b': 3
},
update_task='Test2')
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(project.requires()[0].output().exists())
project.requires()[0].output().remove()
ok_(do_test1)
eq_(a_value, 1)
ok_(do_test2)
eq_(b_value, 3)
def test_create_data_prepare(self):
do_test1 = False
a_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
do_test1 = True
a_value = a
project = create_data_prepare({'Test1': test1}, {'a': 1, 'b': 2})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(do_test1)
eq_(a_value, 1)
def test_create_data_prepare_save_and_load(self):
do_test1 = False
a_value = None
before_value = None
do_test2 = False
b_value = None
def test1(a, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test1
nonlocal a_value
nonlocal before_value
do_test1 = True
a_value = a
with before_artifact_directory.joinpath('aaa').open('r') as f:
before_value = f.read()
def test2(b, artifact_directory=None, before_artifact_directory=None):
nonlocal do_test2
nonlocal b_value
do_test2 = True
b_value = b
with artifact_directory.joinpath('aaa').open('w') as f:
f.write('aaa')
project = create_data_prepare({
'Test2': test2,
'Test1': test1
}, {
'a': 1,
'b': 2
})
run_result = luigi.build([project],
worker_scheduler_factory=DummyFactory())
ok_(run_result)
ok_(project.output().exists())
project.output().remove()
ok_(project.requires()[0].output().exists())
project.requires()[0].output().remove()
ok_(do_test1)
eq_(a_value, 1)
ok_(do_test2)
eq_(b_value, 2)
eq_(before_value, 'aaa')
def __init__(self, *args: Any, **kwargs: Any):
super(RunOnceProject, self).__init__(*args, **kwargs)
self.experiment_id = 0
self.update = True
# parameter preprocessing
with open(self.param_path, 'r') as f:
params = yaml.full_load(f)
self.model_params = params['model']
self.dataset_params = params['dataset']
self.preprocess_params = params['preprocess']
self.parameters = {
'runner': self.runner,
'model': self.model,
'dataset': self.dataset,
'param_path': self.param_path,
**self.model_params,
**self.dataset_params
}
self.run_name = '_'.join([self.runner, self.model, self.dataset])
if 'projects' not in self.preprocess_params:
self.preprocess_params['projects'] = {}
if 'parameters' not in self.preprocess_params:
self.preprocess_params['parameters'] = {}
if 'update_task' not in self.preprocess_params:
self.preprocess_params['update_task'] = ''
self.before_project = create_data_prepare(
{
k: get_attribute(v)
for k, v in self.preprocess_params['projects'].items()
}, self.preprocess_params['parameters'],
self.preprocess_params['update_task'])