def test_deprecated_settings(self):
self.assertRaises(ValueError,
b2luigi.get_setting,
key="my_setting",
deprecated_keys=["my_old_setting"])
b2luigi.set_setting("my_old_setting", "my value")
with warnings.catch_warnings(record=True) as w:
self.assertEqual(
"my value",
b2luigi.get_setting("my_setting",
deprecated_keys=["my_old_setting"]))
self.assertEqual(len(w), 1)
self.assertIsInstance(w[-1].message, DeprecatedSettingsWarning)
self.assertIn("deprecated", str(w[-1].message))
b2luigi.set_setting("my_setting", "my new_value")
with warnings.catch_warnings(record=True) as w:
self.assertEqual(
"my new_value",
b2luigi.get_setting("my_setting",
default="default",
deprecated_keys=["my_old_setting"]))
self.assertEqual(len(w), 0)
def test_set_by_function(self):
self.assertRaises(ValueError, b2luigi.get_setting, "my_setting")
self.assertEqual("default", b2luigi.get_setting("my_setting", "default"))
b2luigi.set_setting("my_setting", "my value")
self.assertEqual("my value", b2luigi.get_setting("my_setting"))
self.assertEqual("my value", b2luigi.get_setting("my_setting", "default"))
def test_set_by_file(self):
with open("settings.json", "w") as f:
json.dump({"my_setting": "my file value"}, f)
self.assertEqual("my file value", b2luigi.get_setting("my_setting"))
b2luigi.set_setting("my_setting", "my value")
self.assertEqual("my value", b2luigi.get_setting("my_setting"))
def setUp(self):
super().setUp()
self.gb2_mock_process = Mock()
self.gb2_mock_process.task = MyGbasf2Task("some_parameter")
self.gb2_mock_process.dirac_user = "******"
self.gb2_mock_process.gbasf2_project_name = get_unique_project_name(
self.gb2_mock_process.task)
self.gb2_mock_process.max_retries = 0
b2luigi.set_setting("gbasf2_print_status_updates", False)
def test_set_by_task_or_file(self):
with open("settings.json", "w") as f:
json.dump({"my_setting": "my file value"}, f)
b2luigi.set_setting("my_second_setting", "my value")
task = b2luigi.Task()
setattr(task, "my_third_setting", "my task value")
self.assertEqual("my file value", b2luigi.get_setting("my_setting", task=task))
self.assertEqual("my value", b2luigi.get_setting("my_second_setting", task=task))
self.assertEqual("my task value", b2luigi.get_setting("my_third_setting", task=task))
def test_set_job_name_via_task_attribute(self):
task = MyTask("some_parameter")
task.job_name = "some_job_name"
submit_file_lines = self._get_htcondor_submit_file_string(
task).splitlines()
self.assertIn("JobBatchName = some_job_name", submit_file_lines)
b2luigi.set_setting("job_name", "some_job_name")
submit_file_lines = self._get_htcondor_submit_file_string(
MyTask("some_parameter")).splitlines()
b2luigi.clear_setting("job_name")
self.assertIn("JobBatchName = some_job_name", submit_file_lines)
def test_set_job_name_is_overriden_by_htcondor_settings(self):
"""
``job_name`` is a global setting, but if the ``JobBatchName`` is set explicitly via the settings, we
want that to override the global setting
"""
task = MyTask("some_parameter")
task.job_name = "job_name_global"
htcondor_settings = {"JobBatchName": "job_name_htcondor"}
b2luigi.set_setting("htcondor_settings", htcondor_settings)
submit_file_lines = self._get_htcondor_submit_file_string(
task).splitlines()
b2luigi.clear_setting("htcondor_settings")
self.assertNotIn("JobBatchName = job_name_global", submit_file_lines)
self.assertIn("JobBatchName = job_name_htcondor", submit_file_lines)
def test_file_path_usage(self):
class TaskA(b2luigi.Task):
some_parameter = b2luigi.IntParameter()
def output(self):
yield self.add_to_output("file_a")
yield self.add_to_output("file_b")
task = TaskA(some_parameter=3)
b2luigi.set_setting("result_dir", "results/some_crazy_path")
self.assertEqual(get_filled_params(task), {"some_parameter": 3})
self.assertFalse(task.get_input_file_names())
self.assertRaises(KeyError, lambda: task._get_input_targets("some_file"))
self.assertEqual(task._get_output_target("file_a").path, task.get_output_file_name("file_a"))
self.assertIn("file_a", task.get_output_file_name("file_a"))
self.assertIn("file_b", task.get_output_file_name("file_b"))
self.assertIn("some_parameter=3", task.get_output_file_name("file_a"))
self.assertIn("some_crazy_path", task.get_output_file_name("file_a"))
import b2luigi
class MyTask(b2luigi.Task):
def output(self):
yield self.add_to_output("test.txt")
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("test.txt"), "w") as f:
f.write("Test")
if __name__ == "__main__":
b2luigi.set_setting("result_dir", "results")
b2luigi.process(MyTask())
def setUp(self):
self.test_dir = tempfile.mkdtemp()
self.cwd = os.getcwd()
os.chdir(self.test_dir)
b2luigi.set_setting("result_dir", "results")
import yaml
import b2luigi
from contre.bootstrapping import DelegateBootstrapping
parameter_file = 'example_parameters.yaml'
with open(parameter_file) as f:
parameters = yaml.load(f)
b2luigi.set_setting(
"result_path",
parameters.get("result_path"),
)
b2luigi.process(
DelegateBootstrapping(name=parameters.get("name"),
parameter_file=parameter_file))
for try_number in range(3):
yield TimingTask(
feature_parameter={feature_name: settings[feature_name]},
n_jobs=job,
try_number=try_number,
num_ids=10,
time_series_length=time_series_length,
random_seed=42
)
def output(self):
yield self.add_to_output("results.csv")
def run(self):
results = []
for input_file in self._get_input_targets("result.json"):
with input_file.open("r") as f:
results.append(json.load(f))
df = pd.DataFrame(results)
with self._get_output_target("results.csv").open("w") as f:
df.to_csv(f)
if __name__ == "__main__":
luigi.set_setting("result_path", "results")
luigi.process(CombinerTask())
import b2luigi
import os
class MyTask(b2luigi.DispatchableTask):
env = {"MY_SECOND_SECRET_VARIABLE": "47"}
def process(self):
print("MY_SECRET_VARIABLE", os.getenv("MY_SECRET_VARIABLE"))
print("MY_SECOND_SECRET_VARIABLE",
os.getenv("MY_SECOND_SECRET_VARIABLE"))
if __name__ == "__main__":
# We cheat a bit and create the setup script here:
with open("setup_script.sh", "w") as f:
f.write("""echo "Setup Script called"\nexport MY_SECRET_VARIABLE=42""")
b2luigi.set_setting("env_script", os.path.abspath("setup_script.sh"))
b2luigi.process(MyTask())
import b2luigi
class MyTask(b2luigi.Task):
some_parameter = b2luigi.Parameter()
def output(self):
yield self.add_to_output("test.txt")
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("test.txt"), "w") as f:
f.write("Test")
@b2luigi.requires(MyTask)
class MyAdditionalTask(b2luigi.Task):
def output(self):
yield self.add_to_output("combined.txt")
@b2luigi.dispatch
@b2luigi.on_temporary_files
def run(self):
with open(self.get_output_file_name("combined.txt"), "w"):
os.kill(os.getpid(), 11)
if __name__ == "__main__":
b2luigi.set_setting("batch_system", "test")
b2luigi.process(MyAdditionalTask(some_parameter="bla_blub"), batch=True)
