def test_get_returns_property_from_loaders(self):
k = util.base36_str()
v = util.base36_str()
loaders = [DictPropertyLoader("some-loader", {k: v})]
props._set_loaders(loaders)
self.assertEqual(v, props.get(k))
def test_get_property_from_list_of_loaders_returns_prop_from_single_loader(
self):
k = util.base36_str()
v = util.base36_str()
loaders = [DictPropertyLoader(util.base36_str(), {k: v})]
ret = props.get_property_from_list_of_loaders(loaders, k)
self.assertEqual(v, ret)
def test_DictPropertyLoader_get_returns_property_from_dict(self):
k = util.base36_str()
v = util.base36_str()
loader = DictPropertyLoader(util.base36_str(), {k: v})
ret = loader.get(k)
self.assertEqual(v, ret)
def test__set_loaders_then_uses_loader_with_get_call(self):
k = util.base36_str()
v = util.base36_str()
loaders = [DictPropertyLoader("some-loader", {k: v})]
props._set_loaders(loaders)
ret = props.get(k)
self.assertEqual(ret, v)
def test_create_file_raises_FileExistsError_if_file_already_exists(self):
source_roots = [tempfile.mkdtemp()]
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
path_already_at_dest = util.base36_str()
self.__create_empty_file(os.path.join(destination_root, path_already_at_dest))
with self.assertRaises(FileExistsError):
generator.create_file(util.base36_str(), path_already_at_dest)
def test_merge_list_of_property_loaders_merges_properties_from_multiple_loaders_as_expected(
self):
k1 = util.base36_str()
v1 = util.base36_str()
k2 = util.base36_str()
v2_a = util.base36_str()
v2_b = util.base36_str()
k3 = util.base36_str()
v3 = util.base36_str()
k4 = util.base36_str()
v4 = util.base36_str()
loaders = [
DictPropertyLoader("first", {
k1: v1,
k2: v2_a
}),
DictPropertyLoader("second", {
k2: v2_b,
k3: v3
}),
DictPropertyLoader("third", {k4: v4}),
]
ret = props.merge_list_of_property_loaders(loaders)
expected_ret = {
k1: v1,
k2: v2_a,
k3: v3,
k4: v4,
}
self.assertEqual(expected_ret, ret)
def test_join_works_with_multiple_fragments(self):
first_overlay_path = tempfile.mkdtemp()
path._set_overlay_paths([first_overlay_path])
frag1 = util.base36_str()
frag2 = util.base36_str()
expected_result = os.path.join(first_overlay_path, frag1, frag2)
actual_result = path.join(frag1, frag2)
self.assertEqual(expected_result, actual_result)
def test_mkdir_makes_parent_dirs(self):
destination_root = tempfile.mkdtemp()
nested_destination_path = os.path.join(util.base36_str(), util.base36_str())
generator = ConfigurableSourceDestGenerator([tempfile.mkdtemp()], destination_root)
generator.mkdir(nested_destination_path)
expected_output_path = os.path.join(destination_root, nested_destination_path)
self.assertTrue(os.path.exists(expected_output_path))
self.assertTrue(os.path.isdir(expected_output_path))
def test_get_property_from_list_of_loaders_overwrites_properties(self):
k = util.base36_str()
v1 = util.base36_str()
v2 = util.base36_str()
loaders = [
DictPropertyLoader("first", {k: v1}),
DictPropertyLoader("second", {k: v2}),
]
ret = props.get_property_from_list_of_loaders(loaders, k)
self.assertEqual(ret, v1)
def test_copy_file_raises_FileExistsError_if_destination_already_exists(self):
source_root = tempfile.mkdtemp()
destination_root = tempfile.mkdtemp()
source_path = util.base36_str()
self.__create_empty_file(os.path.join(source_root, source_path))
destination_path = util.base36_str()
self.__create_empty_file(os.path.join(destination_root, destination_path))
generator = ConfigurableSourceDestGenerator([source_root], destination_root)
with self.assertRaises(FileExistsError):
generator.copy_file(source_path, destination_path)
def test__set_loaders_loads_props_from_workspace_if_previously_set_to_None(
self):
props._set_loaders(None)
with TemporaryWorkspace() as ws:
props_path = os.path.join(ws, lor._constants.WORKSPACE_PROPS)
os.remove(props_path)
k = util.base36_str()
v = util.base36_str()
new_props = {k: v}
with open(props_path, "w") as f:
yaml.dump(new_props, f)
actual_ret = props.get(k)
self.assertEqual(actual_ret, v)
def test_render_template_writes_rendered_template_to_destination_root(self):
source_roots = [tst_helpers.fixture("templates/")]
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
output_filename = util.base36_str()
foo_val = util.base36_str()
generator.render_template("example.txt.jinja2", output_filename, {"foo": foo_val})
expected_output_path = os.path.join(destination_root, output_filename)
self.assertTrue(os.path.exists(expected_output_path))
file_content = util.read_file_to_string(expected_output_path)
self.assertTrue("foo\n", file_content)
def test_YAMLFilePropertyLoader_get_throws_KeyError_if_variable_not_in_YAML_file(
self):
fixture_file = tst_helpers.fixture("properties/variables.yml")
loader = YAMLFilePropertyLoader(fixture_file)
with self.assertRaises(KeyError):
loader.get(util.base36_str())
def test_resolve_all_list_contains_elements_that_do_exist(self):
file_name = util.base36_str()
overlay_with_result = tempfile.mkdtemp()
self.__create_file(os.path.join(overlay_with_result, file_name))
empty_overlay = tempfile.mkdtemp()
another_overlay_with_result = tempfile.mkdtemp()
self.__create_file(os.path.join(another_overlay_with_result, file_name))
path._set_overlay_paths([
overlay_with_result,
empty_overlay,
another_overlay_with_result
])
ws_path = os.path.join(tempfile.mkdtemp(), "ws")
workspace_generator.create(ws_path)
self.__create_file(os.path.join(ws_path, file_name))
workspace._set_path(ws_path)
expected_result = [
os.path.join(overlay_with_result, file_name),
os.path.join(another_overlay_with_result, file_name),
os.path.join(ws_path, file_name),
]
actual_result = path.resolve_all(file_name)
self.assertEqual(expected_result, actual_result)
def recursively_traverse_task(graph, task, parent_task_name=None, previously_traversed_task_names=None):
if previously_traversed_task_names is None:
previously_traversed_task_names = {}
if task in previously_traversed_task_names:
name = previously_traversed_task_names[task]
else:
name = type(task).__name__ + "_" + util.base36_str(3)
previously_traversed_task_names[task] = name
if parent_task_name is not None:
graph.add_edge(parent_task_name, name)
else:
graph.add_node(name)
top_lvl_labels = {name: name}
requirements = task.requires()
if isinstance(requirements, luigi.Task):
labels = recursively_traverse_task(graph, requirements, name, previously_traversed_task_names)
return util.merge(top_lvl_labels, labels)
elif isinstance(requirements, dict):
labels = top_lvl_labels
for k, v in requirements.items():
sub_labels = recursively_traverse_task(graph, v, name, previously_traversed_task_names)
labels = util.merge(labels, sub_labels)
return labels
elif isinstance(requirements, types.GeneratorType):
labels = top_lvl_labels
for task in requirements:
sub_labels = recursively_traverse_task(graph, task, name, previously_traversed_task_names)
labels = util.merge(labels, sub_labels)
else:
return top_lvl_labels
def test_run_install_script_runs_bin_install(self):
ws_path = os.path.join(tempfile.mkdtemp(), "ws")
workspace_generator.create(ws_path)
installer_path = os.path.join(ws_path,
lor._constants.WORKSPACE_INSTALL_BINSTUB)
os.remove(installer_path)
workspace_dir = ws_path
install_path = os.path.join(workspace_dir,
lor._constants.WORKSPACE_INSTALL_BINSTUB)
str_written_by_script = util.base36_str(20)
_, file_containing_str = tempfile.mkstemp()
install_script = """#!/bin/bash
echo {str} > {out}
""".format(str=str_written_by_script, out=file_containing_str)
with open(install_path, "w") as f:
f.write(install_script)
st = os.stat(str(install_path))
os.chmod(install_path, st.st_mode | stat.S_IEXEC)
workspace.run_install_script(workspace_dir)
self.assertTrue(os.path.exists(file_containing_str))
with open(file_containing_str, "r") as f:
self.assertEqual(str_written_by_script, f.readlines()[0].strip())
def test_create_file_creates_file_with_content(self):
source_roots = [tst_helpers.fixture("templates/")]
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
content = util.base36_str()
filename = util.base36_str()
generator.create_file(content, filename)
output_path = os.path.join(destination_root, filename)
self.assertTrue(os.path.exists(output_path))
written_content = util.read_file_to_string(output_path)
self.assertEqual(content, written_content)
def test__set_loaders_to_None_then_raises_RuntimeError_if_not_in_workspace(
self):
workspace._set_path(None)
props._set_loaders(None)
with self.assertRaises(RuntimeError):
props.get(util.base36_str())
def test_render_template_raises_FileNotFound_if_template_does_not_exist(self):
source_roots = [tempfile.mkdtemp()]
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
with self.assertRaises(FileNotFoundError):
generator.render_template("doesnt-exist", os.path.join(destination_root, util.base36_str()), {})
def test_mkdir_is_ok_if_dir_already_exists(self):
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator([tempfile.mkdtemp()], destination_root)
destination_path = util.base36_str()
os.mkdir(os.path.join(destination_root, destination_path))
generator.mkdir(destination_path)
def test_create_file_creates_subdirs_when_required(self):
source_roots = [tempfile.mkdtemp()]
destination_root = tempfile.mkdtemp()
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
provided_content = util.base36_str()
output_relpath = os.path.join(util.base36_str(), util.base36_str())
generator.create_file(provided_content, output_relpath)
expected_output_path = os.path.join(destination_root, output_relpath)
self.assertTrue(os.path.exists(expected_output_path))
written_content = util.read_file_to_string(expected_output_path)
self.assertTrue(provided_content, written_content)
def test_DictPropertyLoader_get_all_returns_same_dict_as_supplied_in_ctor(
self):
d = {"k1": "v1", "k2": "v2"}
loader = DictPropertyLoader(util.base36_str(), d)
ret = loader.get_all()
self.assertEqual(d, ret)
def test_EnsureExistsOnLocalFilesystemTask_fails_if_file_does_not_exist(self):
file_that_does_not_exist = util.base36_str(20)
task = EnsureExistsOnLocalFilesystemTask(path=file_that_does_not_exist)
ran_ok = luigi.build([task], local_scheduler=True)
self.assertFalse(ran_ok)
def test_read_file_to_string_returns_file_contents_as_string(self):
_, path = tempfile.mkstemp()
content = util.base36_str()
with open(path, "w") as f:
f.write(content)
returned_content = util.read_file_to_string(path)
self.assertEqual(content, returned_content)
def test_resolve_returns_result_from_overlay_if_exists_and_in_overlay(self):
overlay1 = tempfile.mkdtemp()
file_name = util.base36_str()
self.__create_file(os.path.join(overlay1, file_name))
path._set_overlay_paths([overlay1])
returned_path = path.resolve(file_name)
self.assertEqual(returned_path, os.path.join(overlay1, file_name))
def test__set_loaders_then_uses_loader_with_get_all_call(self):
k1 = util.base36_str()
v1 = util.base36_str()
k2 = util.base36_str()
v2 = util.base36_str()
loaders = [
DictPropertyLoader("some-loader", {k1: v1}),
DictPropertyLoader("some-other-loader", {k2: v2}),
]
props._set_loaders(loaders)
expected_value = {
k1: v1,
k2: v2,
}
actual_value = props.get_all()
self.assertEqual(expected_value, actual_value)
def test_join_returns_paths_joined_onto_workspace_if_no_overlay_paths_available(self):
path._set_overlay_paths([])
ws_path = os.path.join(tempfile.mkdtemp(), "ws")
workspace_generator.create(ws_path)
workspace._set_path(ws_path)
fragment = util.base36_str()
expected_result = os.path.join(ws_path, fragment)
actual_result = path.join(fragment)
self.assertEqual(expected_result, actual_result)
def test_render_template_raises_FileExistsError_if_destination_already_exists(self):
source_roots = [tst_helpers.fixture("templates/")]
destination_root = tempfile.mkdtemp()
file_already_in_dest = util.base36_str()
self.__create_empty_file(os.path.join(destination_root, file_already_in_dest))
generator = ConfigurableSourceDestGenerator(source_roots, destination_root)
with self.assertRaises(FileExistsError):
generator.render_template("example.txt.jinja2", file_already_in_dest, {})
def test_resolve_returns_result_from_another_overlay_if_exists_in_different_overlay(self):
empty_overlay = tempfile.mkdtemp()
overlay_with_file = tempfile.mkdtemp()
file_name = util.base36_str()
self.__create_file(os.path.join(overlay_with_file, file_name))
path._set_overlay_paths([empty_overlay, overlay_with_file])
returned_path = path.resolve(file_name)
self.assertEqual(os.path.join(overlay_with_file, file_name), returned_path)
def test_write_str_to_file(self):
dir = tempfile.mkdtemp()
dest = os.path.join(dir, "file")
s = util.base36_str()
util.write_str_to_file(dest, s)
self.assertTrue(os.path.exists(dest))
with open(dest, "r") as f:
content = f.read()
self.assertEqual(s, content)