def test_generate_config_report(self) -> None:
config_name = "configuration_name"
timestamp = "1548328077"
build_failed = False
configuration = Configuration(config_name, timestamp, "dbd", False,
Path())
configuration.components["component1"] = ComponentConfig(
DistType.RELEASE, "1.0.0.", "image1", True)
configuration.components["component2"] = ComponentConfig(
DistType.SNAPSHOT, "2.0.0.", "image2", False)
result = dbd.output.output.generate_config_report(
configuration, build_failed)
yaml_dict = yaml.load(result)
self.assertEqual(config_name, yaml_dict.get("name"))
self.assertEqual(timestamp, str(yaml_dict.get("timestamp")))
self.assertEqual(build_failed, not yaml_dict.get("build_successful"))
self.assertEqual(["component1", "component2"],
yaml_dict.get("component-order"))
components = yaml_dict.get("components")
self.assertIsNotNone(components)
component1 = components.get("component1")
self._assert_component_properties(configuration, "component1",
component1)
component2 = components.get("component2")
self._assert_component_properties(configuration, "component2",
component2)
def test_generate_config_report(self) -> None:
config_name = "configuration_name"
timestamp = "1548328077"
build_failed = False
configuration = Configuration(config_name, timestamp, "dbd", False, Path())
configuration.components["component1"] = ComponentConfig(DistType.RELEASE, "1.0.0.", "image1", True)
configuration.components["component2"] = ComponentConfig(DistType.SNAPSHOT, "2.0.0.", "image2", False)
result = dbd.output.output.generate_config_report(configuration, build_failed)
yaml_dict = yaml.load(result)
self.assertEqual(config_name, yaml_dict.get("name"))
self.assertEqual(timestamp, str(yaml_dict.get("timestamp")))
self.assertEqual(build_failed, not yaml_dict.get("build_successful"))
self.assertEqual(["component1", "component2"], yaml_dict.get("component-order"))
components = yaml_dict.get("components")
self.assertIsNotNone(components)
component1 = components.get("component1")
self._assert_component_properties(configuration, "component1", component1)
component2 = components.get("component2")
self._assert_component_properties(configuration, "component2", component2)
def generate_compose_config_file_text(sorted_components: List[str],
configuration: Configuration) -> str:
"""
Generates the contents of the compose-config file.
Args:
sorted_components: The names of the components in the order they were built.
configuration: The `Configuration` object that contains the information about the configuration of the build.
Returns:
The contents of the compose-config file.
"""
text = io.StringIO()
for component in sorted_components:
file_path = configuration.get_compose_config_part(component)
if file_path.exists():
with file_path.open() as file:
contents = file.read()
comment = "# {}\n".format(component)
text.write(comment)
text.write(contents + "\n\n")
return text.getvalue()
def generate_docker_compose_file_text(input_component_config: Dict[str, Any],
configuration: Configuration) -> str:
"""
Generates the contents of the docker-compose file.
Args:
input_component_config: The "components" section of the `BuildConfiguration` dictionary.
configuration: The `Configuration` object that contains the information about the configuration of the build.
Returns:
The contents of the docker-compose file as a string.
"""
docker_compose_parts = {}
for component in input_component_config:
file_path = configuration.get_docker_compose_part(component)
with file_path.open() as file:
docker_compose_part = yaml.load(file)
docker_compose_parts[component] = docker_compose_part
if configuration.kerberos:
krb5 = dbd.defaults.KERBEROS_SERVICE_CONFIG
docker_compose_parts["krb5"] = yaml.load(krb5)
customised_services = {
component: value.get("services", {})
for component, value in input_component_config.items()
}
docker_compose_dict = dbd.output.docker_compose_generator.generate_docker_compose_file_dict(
docker_compose_parts, customised_services)
return yaml.dump(docker_compose_dict, default_style=None)
def generate_config_report(configuration: Configuration, build_failed: bool) -> str:
"""
Generates the contents of the output_configuration.yaml file.
Args:
configuration: The `Configuration` object that contains the information about the configuration of the build.
build_failed: Whether the build failed.
Returns:
The contents of the output_configuration.yaml file.
"""
text = io.StringIO()
text.write("name: {}\n".format(configuration.name))
text.write("timestamp: {}\n".format(configuration.timestamp))
text.write("build_successful: {}\n".format(not build_failed))
text.write("component-order: {}\n".format(configuration.get_component_order()))
text.write("components:\n")
indentation = " "
for component, config in configuration.components.items():
text.write(indentation + component + ":\n")
text.write(indentation * 2 + "dist_type: "
+ ("release" if config.dist_type == DistType.RELEASE else "snapshot")
+ "\n")
text.write(indentation * 2 + "version: " + config.version + "\n")
text.write(indentation * 2 + "image_name: " + config.image_name + "\n")
text.write(indentation * 2 + "reused: " + str(config.reused).lower() + "\n")
return text.getvalue()
def generate_compose_config_file_text(sorted_components: List[str], configuration: Configuration) -> str:
"""
Generates the contents of the compose-config file.
Args:
sorted_components: The names of the components in the order they were built.
configuration: The `Configuration` object that contains the information about the configuration of the build.
Returns:
The contents of the compose-config file.
"""
text = io.StringIO()
for component in sorted_components:
file_path = configuration.get_compose_config_part(component)
if file_path.exists():
with file_path.open() as file:
contents = file.read()
comment = "# {}\n".format(component)
text.write(comment)
text.write(contents + "\n\n")
return text.getvalue()
def generate_docker_compose_file_text(input_component_config: Dict[str, Any], configuration: Configuration) -> str:
"""
Generates the contents of the docker-compose file.
Args:
input_component_config: The "components" section of the `BuildConfiguration` dictionary.
configuration: The `Configuration` object that contains the information about the configuration of the build.
Returns:
The contents of the docker-compose file as a string.
"""
docker_compose_parts = {}
for component in input_component_config:
file_path = configuration.get_docker_compose_part(component)
with file_path.open() as file:
docker_compose_part = yaml.load(file)
docker_compose_parts[component] = docker_compose_part
if configuration.kerberos:
krb5 = dbd.defaults.KERBEROS_SERVICE_CONFIG
docker_compose_parts["krb5"] = yaml.load(krb5)
customised_services = {component : value.get("services", {})
for component, value in input_component_config.items()}
docker_compose_dict = dbd.output.docker_compose_generator.generate_docker_compose_file_dict(docker_compose_parts,
customised_services)
return yaml.dump(docker_compose_dict, default_style=None)
def _get_component_config_and_configuration(
) -> Tuple[Dict[str, Any], Configuration]:
component_config = {"release": "1.0.0"}
configuration = Configuration("configuration_name", "0001", "dbd",
False, Path())
return (component_config, configuration)
def test_generate_compose_config_file_text(self) -> None:
component1_name = "component1"
component2_name = "component2"
compose_config_part1 = """\
CORE-SITE.XML_fs.default.name=hdfs://namenode:9000
CORE-SITE.XML_fs.defaultFS=hdfs://namenode:9000"""
compose_config_part2 = """\
CORE-SITE.XML_hadoop.proxyuser.oozie.hosts=*
CORE-SITE.XML_hadoop.proxyuser.oozie.groups=*"""
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_dir = Path(tmp_dir_name)
TestOutput._write_docker_compose_files(tmp_dir, component1_name,
compose_config_part1)
TestOutput._write_docker_compose_files(tmp_dir, component2_name,
compose_config_part2)
configuration = Configuration("configuration_name", "0001", "dbd",
False, tmp_dir)
result = dbd.output.output.generate_compose_config_file_text(
[component1_name, component2_name], configuration)
lines_component1 = compose_config_part1.split("\n")
lines_component2 = compose_config_part2.split("\n")
input_lines = lines_component1 + lines_component2
output_lines = result.split("\n")
for line in input_lines:
self.assertIn(line, output_lines)
def build(self,
component_config: Dict[str, Any],
built_config: Configuration,
force_rebuild: bool = False) -> ComponentConfig:
(dist_type, argument) = _dist_type_and_arg(component_config)
dist_info = DistInfo(dist_type, argument)
id_string = _get_id_string(dist_info)
image_name = self._get_image_name(id_string, built_config)
docker_context = built_config.get_docker_context(self.name())
pipeline = self._pipeline_builder.build_pipeline(
built_config, component_config, self._assembly, image_name,
dist_info, docker_context)
reused_docker_image: bool
if force_rebuild:
reused_docker_image = False
self._pipeline_executor.execute_all(self.name(), dist_type,
id_string, self._cache,
pipeline)
else:
reused_docker_image = pipeline.final_stage.postcondition_satisfied(
)
self._pipeline_executor.execute_needed(self.name(), dist_type,
id_string, self._cache,
pipeline)
version = self._get_version(dist_type, argument, image_name)
return ComponentConfig(dist_type, version, image_name,
reused_docker_image)
def get_default_arguments(self, kerberos: bool = False) -> Dict[str, Any]:
return {
"built_config" : Configuration("test_configuration_name",
"0001",
"test_repository",
kerberos,
self._tmp_dir_path),
"component_input_config": dict(),
"assembly" : Assembly.from_dict({"url": "some_url"}),
"image_name" : "test_image",
"dist_info" : DistInfo(DistType.SNAPSHOT, "path/to/snapshot_build"),
"docker_context_dir" : Path("path/to/docker/context")
}
def _test_generate_docker_compose_file_text_customised_service(
self, kerberos: bool) -> None:
with tempfile.TemporaryDirectory() as tmp_dir_name:
tmp_dir = Path(tmp_dir_name)
middle_path = "kerberos" if kerberos else "unsecure"
resource_dir = tmp_dir / "hadoop" / middle_path
resource_dir.mkdir(exist_ok=True, parents=True)
docker_compose_part = """\
services:
resourcemanager:
image: ${HADOOP_IMAGE}
nodemanager:
image: ${HADOOP_IMAGE}
"""
with open(resource_dir / "docker-compose_part.yaml",
"w") as output_file:
output_file.write(docker_compose_part)
build_config_file_text = """\
name: oozie500hadoop265
components:
hadoop:
release: 2.6.5
services:
nodemanager:
ports:
- 11000:11000
- 11002:11002"""
input_conf = yaml.load(build_config_file_text)
input_component_config = input_conf["components"]
configuration = Configuration("configuration_name", "0001", "dbd",
kerberos, tmp_dir)
result = dbd.output.output.generate_docker_compose_file_text(
input_component_config, configuration)
result_dict = yaml.load(result)
self.assertEqual(
input_component_config["hadoop"]["services"]["nodemanager"]
["ports"], result_dict["services"]["nodemanager"]["ports"])
self.assertEqual(kerberos, "krb5" in result_dict["services"])
def test_generate_env_file_text(self) -> None:
configuration = Configuration("configuration_name", "0001", "dbd",
False, Path())
dist_type = DistType.RELEASE
version = "1.0.0"
reused = False
for i in range(4):
configuration.components["component{}".format(
i)] = ComponentConfig(dist_type, version, "image{}".format(i),
reused)
expected_env_file_text = """\
COMPONENT0_IMAGE=image0
COMPONENT1_IMAGE=image1
COMPONENT2_IMAGE=image2
COMPONENT3_IMAGE=image3
"""
self.assertEqual(
expected_env_file_text,
dbd.output.output.generate_env_file_text(configuration))
def generate_config_report(configuration: Configuration,
build_failed: bool) -> str:
"""
Generates the contents of the output_configuration.yaml file.
Args:
configuration: The `Configuration` object that contains the information about the configuration of the build.
build_failed: Whether the build failed.
Returns:
The contents of the output_configuration.yaml file.
"""
text = io.StringIO()
text.write("name: {}\n".format(configuration.name))
text.write("timestamp: {}\n".format(configuration.timestamp))
text.write("build_successful: {}\n".format(not build_failed))
text.write("component-order: {}\n".format(
configuration.get_component_order()))
text.write("components:\n")
indentation = " "
for component, config in configuration.components.items():
text.write(indentation + component + ":\n")
text.write(indentation * 2 + "dist_type: " +
("release" if config.dist_type ==
DistType.RELEASE else "snapshot") + "\n")
text.write(indentation * 2 + "version: " + config.version + "\n")
text.write(indentation * 2 + "image_name: " + config.image_name + "\n")
text.write(indentation * 2 + "reused: " + str(config.reused).lower() +
"\n")
return text.getvalue()
def _get_default_configuration_security(kerberos: bool) -> Configuration:
return Configuration(TestConfiguration.NAME,
TestConfiguration.TIMESTAMP,
TestConfiguration.REPOSITORY, kerberos,
TestConfiguration.RESOURCE_PATH)
def _get_default_configuration() -> Configuration:
return Configuration(TestConfiguration.NAME,
TestConfiguration.TIMESTAMP,
TestConfiguration.REPOSITORY, False,
TestConfiguration.RESOURCE_PATH)