import b2luigi
class MyTask(b2luigi.Task):
def output(self):
yield self.add_to_output("test.txt")
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("test.txt"), "w") as f:
f.write("Test")
if __name__ == "__main__":
b2luigi.process(MyTask())
"request_cpus": 1,
"request_memory": "200 MB"
}
def requires(self):
for i in range(10):
yield self.clone(MyNumberTask, some_parameter=i)
def output(self):
yield self.add_to_output("average.txt")
def run(self):
print("I am now starting the average task")
# Build the mean
summed_numbers = 0
counter = 0
for input_file in self.get_input_file_names("output_file.txt"):
with open(input_file, "r") as f:
summed_numbers += float(f.read())
counter += 1
average = summed_numbers / counter
with open(self.get_output_file_name("average.txt"), "w") as f:
f.write(f"{average}\n")
if __name__ == "__main__":
b2luigi.process(MyAverageTask(), workers=200, batch=True)
modularAnalysis.matchMCTruth('B-', path=path)
modularAnalysis.variablesToNtuple(
'D0', [
'M', 'p', 'E', 'useCMSFrame(p)', 'useCMSFrame(E)',
'daughter(0, kaonID)', 'daughter(1, pionID)', 'isSignal',
'mcErrors'
],
filename=self.get_output_file_name("D_n_tuple.root"),
path=path)
modularAnalysis.variablesToNtuple(
'B-', ['Mbc', 'deltaE', 'isSignal', 'mcErrors', 'M'],
filename=self.get_output_file_name("B_n_tuple.root"),
path=path)
return path
def output(self):
yield self.add_to_output("D_n_tuple.root")
yield self.add_to_output("B_n_tuple.root")
class AggregatorTask(Basf2nTupleMergeTask):
n_events = luigi.IntParameter()
def requires(self):
for event_type in SimulationType:
yield self.clone(AnalysisTask, event_type=event_type)
if __name__ == "__main__":
luigi.process(AggregatorTask(n_events=1), workers=4)
import b2luigi
import random
class MyNumberTask(b2luigi.Task):
some_parameter = b2luigi.IntParameter()
def output(self):
yield self.add_to_output("output_file.txt")
def run(self):
random_number = random.random()
with open(self.get_output_file_name("output_file.txt"), "w") as f:
f.write(f"{random_number}\n")
if __name__ == "__main__":
b2luigi.set_setting("result_dir", "results")
b2luigi.process([MyNumberTask(some_parameter=i) for i in range(100)],
workers=200)
def output(self):
yield self.add_to_output("D_ntuple.root")
yield self.add_to_output("B_ntuple.root")
class AnalysisWrapperTask(b2luigi.WrapperTask):
"""
We use the AnalysisWrapperTask to be able to require multiple analyse tasks with
different input datasets and cut values. For each parameter combination, a
different gbasf2 project will be submitted.
"""
def requires(self):
input_dataset = \
"/belle/MC/release-04-01-04/DB00000774/SkimM13ax1/prod00011778/e1003/4S/r00000/mixed/11180100/udst/sub00/"\
"udst_000006_prod00011778_task10020000006.root"
# if you want to iterate over different cuts, just add more values to this list
mbc_lower_cuts = [5.15, 5.2]
for mbc_lower_cut in mbc_lower_cuts:
yield AnalysisTask(
mbc_lower_cut=mbc_lower_cut,
gbasf2_project_name_prefix="luigiExample",
gbasf2_input_dataset=input_dataset,
max_event=100,
)
if __name__ == '__main__':
main_task_instance = AnalysisWrapperTask()
n_gbasf2_tasks = len(list(main_task_instance.requires()))
b2luigi.process(main_task_instance, workers=n_gbasf2_tasks)
import yaml
import b2luigi
from contre.bootstrapping import DelegateBootstrapping
parameter_file = 'example_parameters.yaml'
with open(parameter_file) as f:
parameters = yaml.load(f)
b2luigi.set_setting(
"result_path",
parameters.get("result_path"),
)
b2luigi.process(
DelegateBootstrapping(name=parameters.get("name"),
parameter_file=parameter_file))
modularAnalysis.matchMCTruth('B-', path=path)
modularAnalysis.variablesToNtuple(
'D0', [
'M', 'p', 'E', 'useCMSFrame(p)', 'useCMSFrame(E)',
'daughter(0, kaonID)', 'daughter(1, pionID)', 'isSignal',
'mcErrors'
],
filename=self.get_output_file_name("D_n_tuple.root"),
path=path)
modularAnalysis.variablesToNtuple(
'B-', ['Mbc', 'deltaE', 'isSignal', 'mcErrors', 'M'],
filename=self.get_output_file_name("B_n_tuple.root"),
path=path)
return path
def output(self):
yield self.add_to_output("D_n_tuple.root")
yield self.add_to_output("B_n_tuple.root")
class MasterTask(Basf2nTupleMergeTask):
n_events = luigi.IntParameter()
def requires(self):
for event_type in SimulationType:
yield self.clone(AnalysisTask, event_type=event_type)
if __name__ == "__main__":
luigi.process(MasterTask(n_events=1), workers=4)
import b2luigi
class AlwaysExistingTarget(b2luigi.Target):
"""
Target that always exists
"""
def exists(self):
return True
class DoNothingTask(b2luigi.ExternalTask):
"""
Dummy task that is always already complete and needs nothing to be done.
Useful for testing ``b2luigi.process`` just for its CLI argument parsing
with some dummy task.
"""
def output(self):
return AlwaysExistingTarget()
if __name__ == "__main__":
b2luigi.process(DoNothingTask(), ignore_additional_command_line_args=False)
for try_number in range(3):
yield TimingTask(
feature_parameter={feature_name: settings[feature_name]},
n_jobs=job,
try_number=try_number,
num_ids=10,
time_series_length=time_series_length,
random_seed=42
)
def output(self):
yield self.add_to_output("results.csv")
def run(self):
results = []
for input_file in self._get_input_targets("result.json"):
with input_file.open("r") as f:
results.append(json.load(f))
df = pd.DataFrame(results)
with self._get_output_target("results.csv").open("w") as f:
df.to_csv(f)
if __name__ == "__main__":
luigi.set_setting("result_path", "results")
luigi.process(CombinerTask())
with open(self.get_output_file_name("output_file.txt"), "w") as f:
f.write(f"{random_number}\n")
class MyAverageTask(b2luigi.Task):
def requires(self):
for i in range(100):
yield self.clone(MyNumberTask, some_parameter=i)
def output(self):
yield self.add_to_output("average.txt")
def run(self):
# Build the mean
summed_numbers = 0
counter = 0
for input_file in self.get_input_file_names("output_file.txt"):
with open(input_file, "r") as f:
summed_numbers += float(f.read())
counter += 1
average = summed_numbers / counter
with open(self.get_output_file_name("average.txt"), "w") as f:
f.write(f"{average}\n")
if __name__ == "__main__":
b2luigi.set_setting("result_dir", "results")
b2luigi.process(MyAverageTask(), workers=200)
return summed_numbers / counter
def average_inputs(self, input_file_list):
summed_averages = 0
counter = 0
for input_file in input_file_list:
summed_averages += self.average_file(input_file)
counter += 1
return summed_averages / counter
def run(self):
print(self.get_input_file_names_from_dict("a"))
average_a = self.average_inputs(
self.get_input_file_names_from_dict("a", "random_numbers.txt"))
average_b = self.average_inputs(
self.get_input_file_names_from_dict("b")["random_numbers.txt"])
with open(self.get_output_file_name("combined_average.txt"),
"w") as out_file:
out_file.write(f"{(average_a + average_b)/2}\n")
def output(self):
yield self.add_to_output("combined_average.txt")
if __name__ == "__main__":
luigi.set_setting("result_path", "results")
luigi.process(TaskB(), workers=4)
import b2luigi
class MyTask(b2luigi.Task):
some_parameter = b2luigi.Parameter()
def output(self):
yield self.add_to_output("test.txt")
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("test.txt"), "w") as f:
f.write("Test")
@b2luigi.requires(MyTask)
class MyAdditionalTask(b2luigi.Task):
def output(self):
yield self.add_to_output("combined.txt")
@b2luigi.dispatch
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("combined.txt"), "w"):
os.kill(os.getpid(), 11)
if __name__ == "__main__":
b2luigi.set_setting("batch_system", "test")
b2luigi.process(MyAdditionalTask(some_parameter="bla_blub"), batch=True)
