class TestDoubleSourceChecksumStorage(_TestChecksumStorage):
"""
Tests for `DoubleSourceChecksumStorage`.
"""
def setUp(self):
self.local_storage = MemoryChecksumStorage()
self.remote_storage = MemoryChecksumStorage()
super().setUp()
def create_storage(self) -> ChecksumStorage:
return DoubleSourceChecksumStorage(self.local_storage, self.remote_storage)
def test_get_when_only_in_local(self):
self.local_storage.set_checksum(EXAMPLE_1_CONFIGURATION_ID, EXAMPLE_1_CHECKSUM)
self.assertEqual({EXAMPLE_1_CONFIGURATION_ID: EXAMPLE_1_CHECKSUM}, self.storage.get_all_checksums())
def test_get_when_only_in_external(self):
self.remote_storage.set_checksum(EXAMPLE_1_CONFIGURATION_ID, EXAMPLE_1_CHECKSUM)
self.assertEqual({EXAMPLE_1_CONFIGURATION_ID: EXAMPLE_1_CHECKSUM}, self.storage.get_all_checksums())
def test_get_when_in_local_and_external(self):
self.local_storage.set_checksum(EXAMPLE_1_CONFIGURATION_ID, EXAMPLE_1_CHECKSUM)
self.remote_storage.set_checksum(EXAMPLE_1_CONFIGURATION_ID, EXAMPLE_2_CHECKSUM)
self.assertEqual(EXAMPLE_1_CHECKSUM, self.storage.get_checksum(EXAMPLE_1_CONFIGURATION_ID))
self.assertEqual({EXAMPLE_1_CONFIGURATION_ID: EXAMPLE_1_CHECKSUM}, self.storage.get_all_checksums())
def test_set_only_affects_local(self):
self.storage.set_checksum(EXAMPLE_1_CONFIGURATION_ID, EXAMPLE_1_CHECKSUM)
self.assertEqual({EXAMPLE_1_CONFIGURATION_ID: EXAMPLE_1_CHECKSUM}, self.local_storage.get_all_checksums())
self.assertEqual({}, self.remote_storage.get_all_checksums())
def build_all(self) -> Dict[BuildConfigurationType, BuildResultType]:
"""
Builds all managed images and their managed dependencies.
:return: mapping between built configurations and their associated build result
"""
logger.info("Building all...")
checksum_storage = DoubleSourceChecksumStorage(MemoryChecksumStorage(),
self.checksum_retriever)
all_build_results: Dict[BuildConfigurationType:BuildResultType] = {}
left_to_build: Set[BuildConfigurationType] = set(
self.managed_build_configurations)
while len(left_to_build) != 0:
build_configuration = left_to_build.pop()
assert build_configuration not in all_build_results.keys()
# Build configuration
build_results = self.build(build_configuration,
left_to_build,
_checksum_storage=checksum_storage)
all_build_results.update(build_results)
# Update known configuration checksums
checksums = {
x.identifier: self.checksum_calculator.calculate_checksum(x)
for x in build_results.keys()
}
checksum_storage.set_all_checksums(checksums)
left_to_build = left_to_build - set(build_results.keys())
logger.info(f"Built: {all_build_results}")
return all_build_results
def decode(self, obj_as_json, **kwargs):
parsed_json = super().decode(obj_as_json)
if parsed_json[CHECKSUM_STORAGE_TYPE_PROPERTY] == CHECKSUM_STORAGE_TYPE_VALUE_MAP[MemoryChecksumStorage]:
return MemoryChecksumStorage()
return {
CHECKSUM_STORAGE_TYPE_VALUE_MAP[DiskChecksumStorage]: DiskChecksumStorageJSONDecoder(),
CHECKSUM_STORAGE_TYPE_VALUE_MAP[ConsulChecksumStorage]: ConsulChecksumStorageJSONDecoder()
}[parsed_json[CHECKSUM_STORAGE_TYPE_PROPERTY]].decode(obj_as_json)
def setUp(self):
super().setUp()
self._captured_main = CaptureWrapBuilder(
capture_stdout=True,
capture_exceptions=lambda e: isinstance(
e, SystemExit) and e.code == 0).build(main)
self.build_configurations = BuildConfigurationContainer[
DockerBuildConfiguration](self.create_docker_setup()[1]
for _ in range(3))
self.run_configuration = Configuration(self.build_configurations)
self.pre_built_configuration = list(self.build_configurations)[0]
builder = DockerBuilder(self.build_configurations)
build_result = builder.build(self.pre_built_configuration)
assert len(build_result) == 1
self.run_configuration.checksum_storage = MemoryChecksumStorage({
self.pre_built_configuration.identifier:
builder.checksum_calculator.calculate_checksum(
self.pre_built_configuration)
})
def __init__(self,
docker_build_configurations: BuildConfigurationContainer[
DockerBuildConfiguration] = None,
docker_registries: Iterable[DockerRegistry] = (),
checksum_storage: ChecksumStorage = None):
self.docker_build_configurations = docker_build_configurations if docker_build_configurations is not None \
else BuildConfigurationContainer[DockerBuildConfiguration]()
self.docker_registries = list(docker_registries)
self.checksum_storage = checksum_storage if checksum_storage is not None else MemoryChecksumStorage(
)
def setUp(self):
super().setUp()
self.checksum_storage = MemoryChecksumStorage()
self.configuration = self.create_built_configuration()
self.uploader = self.create_uploader()
def setUp(self):
self.local_storage = MemoryChecksumStorage()
self.remote_storage = MemoryChecksumStorage()
super().setUp()
def create_storage(self) -> ChecksumStorage:
return MemoryChecksumStorage()
def setUp(self):
super().setUp()
self.checksum_storage = MemoryChecksumStorage()
self.docker_builder = DockerBuilder(checksum_retriever=self.checksum_storage)
class TestDockerBuilder(TestWithDockerBuildConfiguration):
"""
Tests for `DockerBuilder`.
"""
def setUp(self):
super().setUp()
self.checksum_storage = MemoryChecksumStorage()
self.docker_builder = DockerBuilder(checksum_retriever=self.checksum_storage)
def test_build_when_dockerfile_is_invalid(self):
_, configuration = self.create_docker_setup(commands=["invalid"])
self.docker_builder.managed_build_configurations.add(configuration)
self.assertRaises(InvalidDockerfileBuildError, self.docker_builder.build, configuration)
def test_build_when_build_fails(self):
_, configuration = self.create_docker_setup(commands=[f"{RUN_DOCKER_COMMAND} exit 1"])
self.docker_builder.managed_build_configurations.add(configuration)
self.assertRaises(BuildStepError, self.docker_builder.build, configuration)
def test_build_when_from_image_is_not_managed(self):
_, configuration = self.create_docker_setup()
assert configuration.identifier not in \
itertools.chain(*(image.tags for image in self.docker_client.images.list()))
self.assertRaises(UnmanagedBuildError, self.docker_builder.build, configuration)
def test_build_when_from_image_is_managed(self):
configurations = self.create_dependent_docker_build_configurations(4)
self.docker_builder.managed_build_configurations.add_all(configurations)
self.docker_builder.managed_build_configurations.add(self.create_docker_setup()[1])
build_results = self.docker_builder.build(configurations[-1])
self.assertCountEqual(
{configuration: configuration.identifier for configuration in configurations}, build_results)
def test_build_when_circular_dependency(self):
configurations = [
self.create_docker_setup(image_name=EXAMPLE_IMAGE_NAME_1, from_image_name=EXAMPLE_IMAGE_NAME_2)[1],
self.create_docker_setup(image_name=EXAMPLE_IMAGE_NAME_2, from_image_name=EXAMPLE_IMAGE_NAME_1)[1]]
self.docker_builder.managed_build_configurations.add_all(configurations)
self.assertRaises(CircularDependencyBuildError, self.docker_builder.build_all)
def test_build_when_up_to_date(self):
_, configuration = self.create_docker_setup()
self.docker_builder.managed_build_configurations.add(configuration)
self.checksum_storage.set_checksum(
configuration.identifier, self.docker_builder.checksum_calculator.calculate_checksum(configuration))
build_results = self.docker_builder.build(configuration)
self.assertEqual(0, len(build_results))
def test_build_all_when_none_managed(self):
built = self.docker_builder.build_all()
self.assertEqual(0, len(built))
def test_build_all_when_one_fails(self):
configurations = self.create_dependent_docker_build_configurations(4)
configurations += [self.create_docker_setup(commands=[f"{RUN_DOCKER_COMMAND} exit 1"])[1]]
self.docker_builder.managed_build_configurations.add_all(configurations)
self.assertRaises(BuildStepError, self.docker_builder.build_all)
def test_build_all_when_managed(self):
configurations = self.create_dependent_docker_build_configurations(4)
self.docker_builder.managed_build_configurations.add_all(configurations)
build_results = self.docker_builder.build_all()
self.assertCountEqual(
{configuration: configuration.identifier for configuration in configurations}, build_results)
def test_build_all_when_some_up_to_date(self):
import logging
logging.getLogger().setLevel(logging.DEBUG)
configurations = self.create_dependent_docker_build_configurations(4)
self.docker_builder.managed_build_configurations.add_all(configurations)
build_results = self.docker_builder.build(configurations[1])
assert len(build_results) == 2
for configuration in build_results.keys():
checksum = self.docker_builder.checksum_calculator.calculate_checksum(configuration)
self.checksum_storage.set_checksum(configuration.identifier, checksum)
build_results = self.docker_builder.build_all()
self.assertCountEqual(configurations[2:], build_results)
def test_build_when_from_image_updated(self):
configurations = self.create_dependent_docker_build_configurations(2)
self.docker_builder.managed_build_configurations.add_all(configurations)
build_results = self.docker_builder.build_all()
assert len(build_results) == 2
for configuration in build_results.keys():
checksum = self.docker_builder.checksum_calculator.calculate_checksum(configuration)
self.checksum_storage.set_checksum(configuration.identifier, checksum)
parent_image = configurations[0]
with open(parent_image.dockerfile_location, "a") as file:
file.write(f"{RUN_DOCKER_COMMAND} echo 1")
parent_image.reload()
build_results = self.docker_builder.build_all()
self.assertEqual(2, len(build_results))
def __init__(
self,
managed_build_configurations: Iterable[BuildConfigurationType] = None,
checksum_retriever: ChecksumRetriever = None,
checksum_calculator_factory: Callable[[],
ChecksumCalculatorType] = None):
"""
Constructor.
:param managed_build_configurations: build configurations that are managed by this builder
:param checksum_retriever: checksum retriever
:param checksum_calculator_factory: callable that returns a checksum calculator
"""
super().__init__(managed_build_configurations)
self.checksum_retriever = checksum_retriever if checksum_retriever is not None else MemoryChecksumStorage(
)
self.checksum_calculator = checksum_calculator_factory()
def build(self, build_configuration: BuildConfigurationType,
allowed_builds: Iterable[BuildConfigurationType]=None, *, _building: Set[BuildConfigurationType]=None,
_checksum_storage: ChecksumStorage=None) \
-> Dict[BuildConfigurationType, BuildResultType]:
"""
Builds the given build configuration, including any (allowed and managed) dependencies.
:param build_configuration: the configuration to build
:param allowed_builds: dependencies that can get built in order to build the configuration. If set
to `None`, all dependencies will be built (default)
:param _building: internal use only (tracks build stack to detect circular dependencies)
:param _checksum_storage: internal use only (has checksums of newly built configurations)
:return: mapping between built configurations and their associated build result
:raises UnmanagedBuildError: when requested to potentially build an unmanaged build
:raises CircularDependencyBuildError: when circular dependency in FROM image
:raises BuildFailedError: raised if the build fails
"""
if build_configuration not in self.managed_build_configurations:
raise UnmanagedBuildError(
f"Build configuration {build_configuration} cannot be built as it is not in the "
f"set of managed build configurations")
building = _building if _building is not None else set()
checksum_storage = _checksum_storage if _checksum_storage is not None else self.checksum_retriever
allowed_builds = set(allowed_builds if allowed_builds is not None else
self.managed_build_configurations)
# Storing checksums of updated dependency builds
checksum_storage = DoubleSourceChecksumStorage(MemoryChecksumStorage(),
checksum_storage)
# Manage collection of what configurations can be built
allowed_builds.add(build_configuration)
if not allowed_builds.issubset(self.managed_build_configurations):
raise UnmanagedBuildError(
f"Allowed builds is not a subset of managed build configurations. Unmanaged builds in `allowed_build`: "
f"{allowed_builds.difference(self.managed_build_configurations)}"
)
if self._already_up_to_date(build_configuration):
return {}
# TODO: Break dependency build into separate function
# Build dependent configurations
build_results: OrderedDict[
BuildConfigurationType:BuildResultType] = OrderedDict()
for required_build_configuration_identifier in build_configuration.requires:
required_build_configuration = self.managed_build_configurations.get(
required_build_configuration_identifier, default=None)
if required_build_configuration in allowed_builds \
and not self._already_up_to_date(required_build_configuration):
left_allowed_builds = allowed_builds - set(
build_results.keys())
if required_build_configuration in building:
raise CircularDependencyBuildError(
f"Circular dependency detected on {required_build_configuration.identifier}"
)
# Build dependency ("parent")
building.add(required_build_configuration)
parent_build_results = self.build(
required_build_configuration,
left_allowed_builds,
_building=building,
_checksum_storage=checksum_storage)
building.remove(required_build_configuration)
# Store dependency build results
assert set(
build_results.keys()).isdisjoint(parent_build_results)
build_results.update(parent_build_results)
# Update known configuration checksums
checksums = {
x.identifier:
self.checksum_calculator.calculate_checksum(x)
for x in build_results.keys()
}
checksum_storage.set_all_checksums(checksums)
# Build main configuration
build_result = self._build(build_configuration)
assert build_configuration not in build_results
build_results[build_configuration] = build_result
assert set(build_results.keys()).issubset(allowed_builds)
return build_results