import b2luigi class MyTask(b2luigi.Task): def output(self): yield self.add_to_output("test.txt") @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("test.txt"), "w") as f: f.write("Test") if __name__ == "__main__": b2luigi.process(MyTask())
"request_cpus": 1, "request_memory": "200 MB" } def requires(self): for i in range(10): yield self.clone(MyNumberTask, some_parameter=i) def output(self): yield self.add_to_output("average.txt") def run(self): print("I am now starting the average task") # Build the mean summed_numbers = 0 counter = 0 for input_file in self.get_input_file_names("output_file.txt"): with open(input_file, "r") as f: summed_numbers += float(f.read()) counter += 1 average = summed_numbers / counter with open(self.get_output_file_name("average.txt"), "w") as f: f.write(f"{average}\n") if __name__ == "__main__": b2luigi.process(MyAverageTask(), workers=200, batch=True)
modularAnalysis.matchMCTruth('B-', path=path) modularAnalysis.variablesToNtuple( 'D0', [ 'M', 'p', 'E', 'useCMSFrame(p)', 'useCMSFrame(E)', 'daughter(0, kaonID)', 'daughter(1, pionID)', 'isSignal', 'mcErrors' ], filename=self.get_output_file_name("D_n_tuple.root"), path=path) modularAnalysis.variablesToNtuple( 'B-', ['Mbc', 'deltaE', 'isSignal', 'mcErrors', 'M'], filename=self.get_output_file_name("B_n_tuple.root"), path=path) return path def output(self): yield self.add_to_output("D_n_tuple.root") yield self.add_to_output("B_n_tuple.root") class AggregatorTask(Basf2nTupleMergeTask): n_events = luigi.IntParameter() def requires(self): for event_type in SimulationType: yield self.clone(AnalysisTask, event_type=event_type) if __name__ == "__main__": luigi.process(AggregatorTask(n_events=1), workers=4)
import b2luigi import random class MyNumberTask(b2luigi.Task): some_parameter = b2luigi.IntParameter() def output(self): yield self.add_to_output("output_file.txt") def run(self): random_number = random.random() with open(self.get_output_file_name("output_file.txt"), "w") as f: f.write(f"{random_number}\n") if __name__ == "__main__": b2luigi.set_setting("result_dir", "results") b2luigi.process([MyNumberTask(some_parameter=i) for i in range(100)], workers=200)
def output(self): yield self.add_to_output("D_ntuple.root") yield self.add_to_output("B_ntuple.root") class AnalysisWrapperTask(b2luigi.WrapperTask): """ We use the AnalysisWrapperTask to be able to require multiple analyse tasks with different input datasets and cut values. For each parameter combination, a different gbasf2 project will be submitted. """ def requires(self): input_dataset = \ "/belle/MC/release-04-01-04/DB00000774/SkimM13ax1/prod00011778/e1003/4S/r00000/mixed/11180100/udst/sub00/"\ "udst_000006_prod00011778_task10020000006.root" # if you want to iterate over different cuts, just add more values to this list mbc_lower_cuts = [5.15, 5.2] for mbc_lower_cut in mbc_lower_cuts: yield AnalysisTask( mbc_lower_cut=mbc_lower_cut, gbasf2_project_name_prefix="luigiExample", gbasf2_input_dataset=input_dataset, max_event=100, ) if __name__ == '__main__': main_task_instance = AnalysisWrapperTask() n_gbasf2_tasks = len(list(main_task_instance.requires())) b2luigi.process(main_task_instance, workers=n_gbasf2_tasks)
import yaml import b2luigi from contre.bootstrapping import DelegateBootstrapping parameter_file = 'example_parameters.yaml' with open(parameter_file) as f: parameters = yaml.load(f) b2luigi.set_setting( "result_path", parameters.get("result_path"), ) b2luigi.process( DelegateBootstrapping(name=parameters.get("name"), parameter_file=parameter_file))
modularAnalysis.matchMCTruth('B-', path=path) modularAnalysis.variablesToNtuple( 'D0', [ 'M', 'p', 'E', 'useCMSFrame(p)', 'useCMSFrame(E)', 'daughter(0, kaonID)', 'daughter(1, pionID)', 'isSignal', 'mcErrors' ], filename=self.get_output_file_name("D_n_tuple.root"), path=path) modularAnalysis.variablesToNtuple( 'B-', ['Mbc', 'deltaE', 'isSignal', 'mcErrors', 'M'], filename=self.get_output_file_name("B_n_tuple.root"), path=path) return path def output(self): yield self.add_to_output("D_n_tuple.root") yield self.add_to_output("B_n_tuple.root") class MasterTask(Basf2nTupleMergeTask): n_events = luigi.IntParameter() def requires(self): for event_type in SimulationType: yield self.clone(AnalysisTask, event_type=event_type) if __name__ == "__main__": luigi.process(MasterTask(n_events=1), workers=4)
import b2luigi class AlwaysExistingTarget(b2luigi.Target): """ Target that always exists """ def exists(self): return True class DoNothingTask(b2luigi.ExternalTask): """ Dummy task that is always already complete and needs nothing to be done. Useful for testing ``b2luigi.process`` just for its CLI argument parsing with some dummy task. """ def output(self): return AlwaysExistingTarget() if __name__ == "__main__": b2luigi.process(DoNothingTask(), ignore_additional_command_line_args=False)
for try_number in range(3): yield TimingTask( feature_parameter={feature_name: settings[feature_name]}, n_jobs=job, try_number=try_number, num_ids=10, time_series_length=time_series_length, random_seed=42 ) def output(self): yield self.add_to_output("results.csv") def run(self): results = [] for input_file in self._get_input_targets("result.json"): with input_file.open("r") as f: results.append(json.load(f)) df = pd.DataFrame(results) with self._get_output_target("results.csv").open("w") as f: df.to_csv(f) if __name__ == "__main__": luigi.set_setting("result_path", "results") luigi.process(CombinerTask())
with open(self.get_output_file_name("output_file.txt"), "w") as f: f.write(f"{random_number}\n") class MyAverageTask(b2luigi.Task): def requires(self): for i in range(100): yield self.clone(MyNumberTask, some_parameter=i) def output(self): yield self.add_to_output("average.txt") def run(self): # Build the mean summed_numbers = 0 counter = 0 for input_file in self.get_input_file_names("output_file.txt"): with open(input_file, "r") as f: summed_numbers += float(f.read()) counter += 1 average = summed_numbers / counter with open(self.get_output_file_name("average.txt"), "w") as f: f.write(f"{average}\n") if __name__ == "__main__": b2luigi.set_setting("result_dir", "results") b2luigi.process(MyAverageTask(), workers=200)
return summed_numbers / counter def average_inputs(self, input_file_list): summed_averages = 0 counter = 0 for input_file in input_file_list: summed_averages += self.average_file(input_file) counter += 1 return summed_averages / counter def run(self): print(self.get_input_file_names_from_dict("a")) average_a = self.average_inputs( self.get_input_file_names_from_dict("a", "random_numbers.txt")) average_b = self.average_inputs( self.get_input_file_names_from_dict("b")["random_numbers.txt"]) with open(self.get_output_file_name("combined_average.txt"), "w") as out_file: out_file.write(f"{(average_a + average_b)/2}\n") def output(self): yield self.add_to_output("combined_average.txt") if __name__ == "__main__": luigi.set_setting("result_path", "results") luigi.process(TaskB(), workers=4)
import b2luigi class MyTask(b2luigi.Task): some_parameter = b2luigi.Parameter() def output(self): yield self.add_to_output("test.txt") @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("test.txt"), "w") as f: f.write("Test") @b2luigi.requires(MyTask) class MyAdditionalTask(b2luigi.Task): def output(self): yield self.add_to_output("combined.txt") @b2luigi.dispatch @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("combined.txt"), "w"): os.kill(os.getpid(), 11) if __name__ == "__main__": b2luigi.set_setting("batch_system", "test") b2luigi.process(MyAdditionalTask(some_parameter="bla_blub"), batch=True)