-
Notifications
You must be signed in to change notification settings - Fork 0
/
pipeline.py
49 lines (40 loc) · 1.44 KB
/
pipeline.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
import luigi
from luigi import configuration
import os
from src.tasks.ensemble_eval import EnsembleEval
from src.tasks.cv_eval import CVEval
import src.tasks.pipeline_funs as pf
import logging
import logging.config
logging_conf = configuration.get_config().get("core", "logging_conf_file")
logging.config.fileConfig(logging_conf)
logger = logging.getLogger("microbiome.pred")
class MicrobiomePred(luigi.WrapperTask):
conf = configuration.get_config()
def requires(self):
ps_path = os.path.join(
self.conf.get("paths", "project_dir"),
self.conf.get("paths", "phyloseq")
)
ensemble = pf.values_from_conf(self.conf, "ensemble")
tasks = []
for i in ensemble.keys():
tasks.append(EnsembleEval(i))
exper = pf.values_from_conf(self.conf, "experiment")
for i in exper.keys():
for k in ["all", "all-cv"] + list(range(1, exper[i]["k_folds"])):
tasks.append(
CVEval(
ps_path,
exper[i]["preprocessing"],
str(exper[i]["validation_prop"]),
str(exper[i]["k_folds"]),
exper[i]["features"],
exper[i]["model"],
str(k),
exper[i]["metrics"]
)
)
return tasks
if __name__ == "__main__":
luigi.run()