def get_image_state(
self, source_image_target: DockerImageTarget,
target_image_target: DockerImageTarget,
image_info_of_dependencies: Dict[str, ImageInfo]) -> ImageState:
if self.is_rebuild_necessary(image_info_of_dependencies):
return ImageState.NEEDS_TO_BE_BUILD
else:
if target_image_target.exists() \
and not build_config().force_pull \
and not build_config().force_load:
self.logger.info(
f"Task {self.__repr__()}: Checking if target image {target_image_target.get_complete_name()} "
f"is locally available, result {target_image_target.exists()}"
)
return ImageState.TARGET_LOCALLY_AVAILABLE
if source_image_target.exists() \
and not build_config().force_pull \
and not build_config().force_load:
self.logger.info(
f"Task {self.__repr__()}: Checking if source image {source_image_target.get_complete_name()} "
f"is locally available, result {source_image_target.exists()}"
)
return ImageState.SOURCE_LOCALLY_AVAILABLE
elif self.can_image_be_loaded(source_image_target):
return ImageState.CAN_BE_LOADED
elif self.is_image_in_registry(source_image_target):
return ImageState.REMOTE_AVAILABLE
else:
return ImageState.NEEDS_TO_BE_BUILD
def _prepare_outputs(self):
self._environment_info_target = luigi.LocalTarget(
"%s/info/environment/%s/environment_info"
% (build_config().output_directory,
self.environment_name))
if self._environment_info_target.exists():
self._environment_info_target.remove()
def _prepare_outputs(self):
self._image_info_target = luigi.LocalTarget(
"%s/info/image/analyze/%s/%s" %
(build_config().output_directory, self._target_repository_name,
self._source_image_tag))
if self._image_info_target.exists():
self._image_info_target.remove()
def _prepare_outputs(self):
self._target = luigi.LocalTarget(
"%s/uploads/%s/%s" %
(build_config().output_directory,
flavor.get_name_from_path(self.flavor_path), self.release_type))
if self._target.exists():
self._target.remove()
def build(self, image_info: ImageInfo):
log_file_path = self.prepare_log_file_path(image_info)
self.logger.info(
"Task %s: Build docker image %s, log file can be found here %s",
self._task_id, image_info.get_target_complete_name(),
log_file_path)
temp_directory = tempfile.mkdtemp(
prefix="script_langauge_container_tmp_dir",
dir=build_config().temporary_base_directory)
try:
image_description = image_info.image_description
build_context_creator = BuildContextCreator(
temp_directory, image_info, log_file_path)
build_context_creator.prepare_build_context_to_temp_dir()
output_generator = \
self._low_level_client.build(path=temp_directory,
nocache=docker_client_config.no_cache,
tag=image_info.get_target_complete_name(),
rm=True,
buildargs=dict(**image_description.transparent_build_arguments,
**image_description.image_changing_build_arguments,
**docker_build_arguments().secret))
self._handle_output(output_generator, image_info, log_file_path)
finally:
shutil.rmtree(temp_directory)
def run_task(self):
image_info_of_dependencies = DependencyImageInfoCollector(
).get_from_dict_of_inputs(self.input())
image_hash = self._build_context_hasher.generate_image_hash(
image_info_of_dependencies)
image_info = ImageInfo(
source_repository_name=self._source_repository_name,
target_repository_name=self._target_repository_name,
source_tag=self._source_image_tag,
target_tag=self._target_image_tag,
hash=image_hash,
commit=self.get_commit_id(),
build_name=build_config().build_name,
depends_on_images=image_info_of_dependencies,
image_state=None,
image_description=self.image_description)
target_image_target = DockerImageTarget(
self._target_repository_name, image_info.get_target_complete_tag())
source_image_target = DockerImageTarget(
self._source_repository_name, image_info.get_source_complete_tag())
image_state = self.get_image_state(source_image_target,
target_image_target,
image_info_of_dependencies)
image_info.image_state = image_state.name # TODO setter for image_state
with self._image_info_target.open("w") as f:
f.write(image_info.to_json())
def _prepare_outputs(self):
self._log_target = luigi.LocalTarget(
"%s/logs/environment/%s/populate_data/%s" %
(build_config().output_directory, self._test_environment_info.name,
self.task_id))
if self._log_target.exists():
self._log_target.remove()
def _prepare_outputs(self):
self._database_ready_target = luigi.LocalTarget(
"%s/info/environment/%s/database/%s/%s/ready" %
(build_config().output_directory, self.environment_name,
self._database_info.container_info.container_name, self.attempt))
if self._database_ready_target.exists():
self._database_ready_target.remove()
def _prepare_outputs(self):
self._test_container_info_target = luigi.LocalTarget(
"%s/info/environment/%s/test-container/%s/%s/container_info" %
(build_config().output_directory, self.environment_name,
self.test_container_name, self.attempt))
if self._test_container_info_target.exists():
self._test_container_info_target.remove()
def _prepare_outputs(self):
self._log_target = luigi.LocalTarget(
"%s/logs/clean/images/%s_%s" %
(build_config().output_directory,
datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'),
self.task_id))
if self._log_target.exists():
self._log_target.remove()
def _prepare_outputs(self):
self._image_info_target = luigi.LocalTarget(
"%s/info/image/build/%s/%s" %
(build_config().output_directory,
self.image_info.target_repository_name,
self.image_info.get_target_complete_tag()))
if self._image_info_target.exists():
self._image_info_target.remove()
def prepare_log_dir(self, release_image_name: str):
log_dir = pathlib.Path("%s/logs/exports/%s/%s/"
% (build_config().output_directory,
release_image_name,
datetime.now().strftime('%Y_%m_%d_%H_%M_%S')))
if log_dir.exists():
shutil.rmtree(log_dir)
log_dir.mkdir(parents=True, exist_ok=True)
return log_dir
def _prepare_outputs(self):
self.flavor_name = flavor.get_name_from_path(str(self.flavor_path))
self.log_path = "%s/logs/test-runner/db-test/tests/%s/%s/%s/" % (
build_config().output_directory, self.flavor_name,
self.release_type,
datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'))
self._status_target = luigi.LocalTarget(self.log_path + "status.log")
if self._status_target.exists():
self._status_target.remove()
def create_dependencies_file(self):
self.dependencies_dir = pathlib.Path(
build_config().output_directory).joinpath("dependencies")
self.dependencies_task_dir = self.dependencies_dir.joinpath(
self.task_id)
self.dependencies_task_dir.mkdir(parents=True, exist_ok=True)
self.dependencies_requires_file = self.dependencies_task_dir.joinpath(
"requires.json")
self.dependencies_dynamic_file = self.dependencies_task_dir.joinpath(
"dynamic.json")
def run_task(self):
build_tasks = self.create_build_tasks_for_all_flavors(
not build_config().force_rebuild)
tasks_creator = ExportContainerTasksCreator(
export_path=self.export_path, release_name=self.release_name)
export_tasks = tasks_creator.create_export_tasks_for_flavors(
build_tasks)
exports_for_flavors = yield export_tasks
self.write_command_line_output(exports_for_flavors)
self.write_output(exports_for_flavors)
def prepate_log_file_path(self, image_info: ImageInfo):
log_file_path = pathlib.Path(
"%s/logs/docker-push/%s/%s/%s" %
(build_config().output_directory,
image_info.target_repository_name, image_info.target_tag,
datetime.now().strftime('%Y_%m_%d_%H_%M_%S')))
log_file_path = luigi.LocalTarget(str(log_file_path))
if log_file_path.exists():
log_file_path.remove()
return log_file_path
def can_image_be_loaded(self, image_target: DockerImageTarget):
if build_config().cache_directory is not None:
image_path = self.get_path_to_cached_image(image_target)
self.logger.info(
f"Task {self.__repr__()}: Checking if image archive {image_path} "
f"is available in cache directory, "
f"result {image_path.exists()}")
return image_path.exists()
else:
return False
def _prepare_outputs(self):
self.flavor_name = flavor.get_name_from_path(self.flavor_path)
self._target = luigi.LocalTarget(
"%s/info/export/%s/%s"
% (build_config().output_directory,
self.flavor_name,
self.release_type
# datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
))
if self._target.exists():
self._target.remove()
def prepare_log_file_path(self, image_info: ImageInfo):
log_file_path = pathlib.Path("%s/logs/docker-build/%s/%s/%s_%s"
% (build_config().output_directory,
image_info.target_repository_name, image_info.target_tag,
datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S'),
image_info.hash
))
if log_file_path.exists():
shutil.rmtree(log_file_path)
log_file_path.mkdir(parents=True)
return log_file_path
def run_task(self):
image_archive_path = Path(build_config().cache_directory) \
.joinpath(self.image_info.get_source_complete_name() + ".tar")
self.logger.info("Task %s: Try to load docker image %s from %s",
self.task_id,
self.image_info.get_source_complete_name(),
image_archive_path)
with image_archive_path.open("rb") as f:
self.client.images.load(f)
self.client.images.get(self.image_info.get_source_complete_name()).tag(
repository=self.image_info.target_repository_name,
tag=self.image_info.get_target_complete_tag())
def create_release(self, release_image_name: str, release_file: str):
self.logger.info("Task %s: Create container file %s", self.__repr__(), release_file)
temp_directory = tempfile.mkdtemp(prefix="release_archive_",
dir=build_config().temporary_base_directory)
try:
log_path = self.prepare_log_dir(release_image_name)
export_file = self.create_and_export_container(release_image_name, temp_directory)
extract_dir = self.extract_exported_container(log_path, export_file, temp_directory)
self.modify_extracted_container(extract_dir)
self.pack_release_file(log_path, extract_dir, release_file)
finally:
shutil.rmtree(temp_directory)
def _log_build_context(self):
if build_config().log_build_context_content:
build_context_log_file = self._log_file_path.joinpath(
"docker-build-context.log")
with build_context_log_file.open("wt") as log_file:
for file in self._get_files_in_build_context(
self._temp_directory):
log_file.write(file)
log_file.write("\n")
dockerfile_log_file = self._log_file_path.joinpath(
"Dockerfile.generated")
shutil.copy(self._temp_directory + "/Dockerfile",
str(dockerfile_log_file))
def get_image_state(
self, source_image_target: DockerImageTarget,
target_image_target: DockerImageTarget,
image_info_of_dependencies: Dict[str, ImageInfo]) -> ImageState:
if self.is_rebuild_necessary(image_info_of_dependencies):
return ImageState.NEEDS_TO_BE_BUILD
else:
if self.is_image_locally_available(target_image_target) \
and not build_config().force_pull \
and not build_config().force_load:
return ImageState.TARGET_LOCALLY_AVAILABLE
if self.is_image_locally_available(source_image_target) \
and not build_config().force_pull \
and not build_config().force_load:
return ImageState.SOURCE_LOCALLY_AVAILABLE
elif self.can_image_be_loaded(source_image_target):
return ImageState.CAN_BE_LOADED
elif self.is_image_in_registry(source_image_target):
return ImageState.REMOTE_AVAILABLE
else:
return ImageState.NEEDS_TO_BE_BUILD
def run_task(self):
test_container = self._client.containers.get(
self._test_container_info.container_name)
db_container_name = self._database_info.container_info.container_name
db_container = self._client.containers.get(db_container_name)
database_log_path = \
pathlib.Path("%s/logs/environment/%s/database/%s/"
% (build_config().output_directory,
self.environment_name,
db_container_name))
is_database_ready = \
self.wait_for_database_startup(database_log_path, test_container, db_container)
after_startup_db_log_file = database_log_path.joinpath(
"after_startup_db_log.tar.gz")
self.save_db_log_files_as_gzip_tar(after_startup_db_log_file,
db_container)
self.write_output(is_database_ready)
class ExportContainers(DockerFlavorBuildBase):
command_line_output_target = \
luigi.LocalTarget("%s/exports/command_line_output" % (build_config().output_directory))
release_types = luigi.ListParameter(["Release"])
export_path = luigi.OptionalParameter(None)
release_name = luigi.OptionalParameter(None)
# TOOD force export
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._prepare_outputs()
def get_goals(self):
self.actual_release_types = [
ReleaseType[release_type] for release_type in self.release_types
]
release_type_goal_map = {
ReleaseType.Release: "release",
ReleaseType.BaseTest: "base_test_build_run",
ReleaseType.FlavorTest: "flavor_test_build_run"
}
goals = [
release_type_goal_map[release_type]
for release_type in self.actual_release_types
]
return goals
def _prepare_outputs(self):
self._export_info_target = luigi.LocalTarget(
"%s/info/exports/final" % (build_config().output_directory))
if self._export_info_target.exists():
self._export_info_target.remove()
def output(self):
return self._export_info_target
def run_task(self):
build_tasks = self.create_build_tasks_for_all_flavors(
not build_config().force_rebuild)
tasks_creator = ExportContainerTasksCreator(
export_path=self.export_path, release_name=self.release_name)
export_tasks = tasks_creator.create_export_tasks_for_flavors(
build_tasks)
exports_for_flavors = yield export_tasks
self.write_command_line_output(exports_for_flavors)
self.write_output(exports_for_flavors)
def write_output(self, exports_for_flavors):
result = {}
for flavor_path, releases in exports_for_flavors.items():
result[flavor_path] = {}
for release_name, release_info_dict in releases.items():
export_info = DependencyExportInfoCollector(
).get_from_sinlge_input(release_info_dict)
result[flavor_path][release_name] = export_info
with self.output().open("w") as f:
json = self.json_pickle_result(result)
f.write(json)
def json_pickle_result(self, result):
jsonpickle.set_preferred_backend('simplejson')
jsonpickle.set_encoder_options('simplejson', sort_keys=True, indent=4)
json = jsonpickle.encode(result)
return json
def write_command_line_output(self, exports_for_flavors):
if self.command_line_output_target.exists():
self.command_line_output_target.remove()
with self.command_line_output_target.open("w") as out_file:
for flavor_path, releases in exports_for_flavors.items():
for release_name, release_info_dict in releases.items():
export_info = DependencyExportInfoCollector(
).get_from_sinlge_input(release_info_dict)
out_file.write("Cached container under %s" %
export_info.cache_file)
out_file.write("\n")
out_file.write("\n")
if export_info.output_file is not None:
out_file.write("Copied container to %s" %
export_info.output_file)
out_file.write("\n")
out_file.write("\n")
out_file.write(
"=================================================")
out_file.write("\n")
def create_timer_base_directories(self):
self.timers_dir = pathlib.Path(
build_config().output_directory).joinpath("timers")
self.timers_state_dir = self.timers_dir.joinpath("state")
self.timers_result_dir = self.timers_dir.joinpath("results")
def _prepare_outputs(self):
self._export_info_target = luigi.LocalTarget(
"%s/info/exports/final" % (build_config().output_directory))
if self._export_info_target.exists():
self._export_info_target.remove()
def remove_release_file_if_requested(self, release_file):
if release_file.exists() and \
(build_config().force_rebuild or
build_config().force_pull):
self.logger.info("Task %s: Removed container file %s", self.__repr__(), release_file)
os.remove(release_file)
def output(self):
self.directory = "%s/exports/" % build_config().output_directory
release_directory = luigi.LocalTarget(self.directory + ".created")
return release_directory
def _prepare_outputs(self):
self._save_info_target = luigi.LocalTarget(
"%s/info/save/%s"
% (build_config().output_directory, self.task_id))
if self._save_info_target.exists():
self._save_info_target.remove()
