def test_deprecated_settings(self): self.assertRaises(ValueError, b2luigi.get_setting, key="my_setting", deprecated_keys=["my_old_setting"]) b2luigi.set_setting("my_old_setting", "my value") with warnings.catch_warnings(record=True) as w: self.assertEqual( "my value", b2luigi.get_setting("my_setting", deprecated_keys=["my_old_setting"])) self.assertEqual(len(w), 1) self.assertIsInstance(w[-1].message, DeprecatedSettingsWarning) self.assertIn("deprecated", str(w[-1].message)) b2luigi.set_setting("my_setting", "my new_value") with warnings.catch_warnings(record=True) as w: self.assertEqual( "my new_value", b2luigi.get_setting("my_setting", default="default", deprecated_keys=["my_old_setting"])) self.assertEqual(len(w), 0)
def test_set_by_function(self): self.assertRaises(ValueError, b2luigi.get_setting, "my_setting") self.assertEqual("default", b2luigi.get_setting("my_setting", "default")) b2luigi.set_setting("my_setting", "my value") self.assertEqual("my value", b2luigi.get_setting("my_setting")) self.assertEqual("my value", b2luigi.get_setting("my_setting", "default"))
def test_set_by_file(self): with open("settings.json", "w") as f: json.dump({"my_setting": "my file value"}, f) self.assertEqual("my file value", b2luigi.get_setting("my_setting")) b2luigi.set_setting("my_setting", "my value") self.assertEqual("my value", b2luigi.get_setting("my_setting"))
def setUp(self): super().setUp() self.gb2_mock_process = Mock() self.gb2_mock_process.task = MyGbasf2Task("some_parameter") self.gb2_mock_process.dirac_user = "******" self.gb2_mock_process.gbasf2_project_name = get_unique_project_name( self.gb2_mock_process.task) self.gb2_mock_process.max_retries = 0 b2luigi.set_setting("gbasf2_print_status_updates", False)
def test_set_by_task_or_file(self): with open("settings.json", "w") as f: json.dump({"my_setting": "my file value"}, f) b2luigi.set_setting("my_second_setting", "my value") task = b2luigi.Task() setattr(task, "my_third_setting", "my task value") self.assertEqual("my file value", b2luigi.get_setting("my_setting", task=task)) self.assertEqual("my value", b2luigi.get_setting("my_second_setting", task=task)) self.assertEqual("my task value", b2luigi.get_setting("my_third_setting", task=task))
def test_set_job_name_via_task_attribute(self): task = MyTask("some_parameter") task.job_name = "some_job_name" submit_file_lines = self._get_htcondor_submit_file_string( task).splitlines() self.assertIn("JobBatchName = some_job_name", submit_file_lines) b2luigi.set_setting("job_name", "some_job_name") submit_file_lines = self._get_htcondor_submit_file_string( MyTask("some_parameter")).splitlines() b2luigi.clear_setting("job_name") self.assertIn("JobBatchName = some_job_name", submit_file_lines)
def test_set_job_name_is_overriden_by_htcondor_settings(self): """ ``job_name`` is a global setting, but if the ``JobBatchName`` is set explicitly via the settings, we want that to override the global setting """ task = MyTask("some_parameter") task.job_name = "job_name_global" htcondor_settings = {"JobBatchName": "job_name_htcondor"} b2luigi.set_setting("htcondor_settings", htcondor_settings) submit_file_lines = self._get_htcondor_submit_file_string( task).splitlines() b2luigi.clear_setting("htcondor_settings") self.assertNotIn("JobBatchName = job_name_global", submit_file_lines) self.assertIn("JobBatchName = job_name_htcondor", submit_file_lines)
def test_file_path_usage(self): class TaskA(b2luigi.Task): some_parameter = b2luigi.IntParameter() def output(self): yield self.add_to_output("file_a") yield self.add_to_output("file_b") task = TaskA(some_parameter=3) b2luigi.set_setting("result_dir", "results/some_crazy_path") self.assertEqual(get_filled_params(task), {"some_parameter": 3}) self.assertFalse(task.get_input_file_names()) self.assertRaises(KeyError, lambda: task._get_input_targets("some_file")) self.assertEqual(task._get_output_target("file_a").path, task.get_output_file_name("file_a")) self.assertIn("file_a", task.get_output_file_name("file_a")) self.assertIn("file_b", task.get_output_file_name("file_b")) self.assertIn("some_parameter=3", task.get_output_file_name("file_a")) self.assertIn("some_crazy_path", task.get_output_file_name("file_a"))
import b2luigi class MyTask(b2luigi.Task): def output(self): yield self.add_to_output("test.txt") @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("test.txt"), "w") as f: f.write("Test") if __name__ == "__main__": b2luigi.set_setting("result_dir", "results") b2luigi.process(MyTask())
def setUp(self): self.test_dir = tempfile.mkdtemp() self.cwd = os.getcwd() os.chdir(self.test_dir) b2luigi.set_setting("result_dir", "results")
import yaml import b2luigi from contre.bootstrapping import DelegateBootstrapping parameter_file = 'example_parameters.yaml' with open(parameter_file) as f: parameters = yaml.load(f) b2luigi.set_setting( "result_path", parameters.get("result_path"), ) b2luigi.process( DelegateBootstrapping(name=parameters.get("name"), parameter_file=parameter_file))
for try_number in range(3): yield TimingTask( feature_parameter={feature_name: settings[feature_name]}, n_jobs=job, try_number=try_number, num_ids=10, time_series_length=time_series_length, random_seed=42 ) def output(self): yield self.add_to_output("results.csv") def run(self): results = [] for input_file in self._get_input_targets("result.json"): with input_file.open("r") as f: results.append(json.load(f)) df = pd.DataFrame(results) with self._get_output_target("results.csv").open("w") as f: df.to_csv(f) if __name__ == "__main__": luigi.set_setting("result_path", "results") luigi.process(CombinerTask())
import b2luigi import os class MyTask(b2luigi.DispatchableTask): env = {"MY_SECOND_SECRET_VARIABLE": "47"} def process(self): print("MY_SECRET_VARIABLE", os.getenv("MY_SECRET_VARIABLE")) print("MY_SECOND_SECRET_VARIABLE", os.getenv("MY_SECOND_SECRET_VARIABLE")) if __name__ == "__main__": # We cheat a bit and create the setup script here: with open("setup_script.sh", "w") as f: f.write("""echo "Setup Script called"\nexport MY_SECRET_VARIABLE=42""") b2luigi.set_setting("env_script", os.path.abspath("setup_script.sh")) b2luigi.process(MyTask())
import b2luigi class MyTask(b2luigi.Task): some_parameter = b2luigi.Parameter() def output(self): yield self.add_to_output("test.txt") @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("test.txt"), "w") as f: f.write("Test") @b2luigi.requires(MyTask) class MyAdditionalTask(b2luigi.Task): def output(self): yield self.add_to_output("combined.txt") @b2luigi.dispatch @b2luigi.on_temporary_files def run(self): with open(self.get_output_file_name("combined.txt"), "w"): os.kill(os.getpid(), 11) if __name__ == "__main__": b2luigi.set_setting("batch_system", "test") b2luigi.process(MyAdditionalTask(some_parameter="bla_blub"), batch=True)