def __init__(self, debug=False, db_path=None, verbose=False, init_logging=True):
"""
:param debug: bool, provide debug output if True
:param db_path: str, path to json file where the database stores the data persistently
:param verbose: bool, print verbose output
:param init_logging: bool, set up logging if True
"""
if init_logging:
self.set_logging(debug=debug, verbose=verbose)
self.verbose = verbose
self.debug = debug
self.db = Database(db_path=db_path)
self.db_path = self.db.db_root_path
def test_backwards_compat(tmpdir):
"""
we keep adding new fields in DB: this test makes sure that none of them are required
so that old config is forward compat with new versions of ab
"""
db_dir_path = str(tmpdir)
db = Database(db_path=db_dir_path)
db_path = db._db_path()
db_content = {
"next_build_id": 2,
"builds": {
"1": {
"build_id":
"1",
"playbook_path":
"playbook.yaml",
"build_volumes": [],
"build_user":
None,
"metadata": {
"working_dir": "/src",
"labels": {},
"annotations": {},
"env_vars": {},
"cmd": None,
"user": None,
"ports": [],
"volumes": []
},
"state":
"done",
"build_start_time":
"20190923-153518169396",
"build_finished_time":
"20190923-153531854630",
"base_image":
"fedora:30",
"target_image":
"ansiblefest-image",
"builder_name":
"buildah",
"layers": [{
"content": None,
"layer_id":
"e9ed59d2baf72308f3a811ebc49ff3f4e0175abf40bf636bea0160759c637999",
"base_image_id": None,
"cached": True
}, {
"content":
"730ecc32518d080377233c10f42ec832f3834cc933ff42a32cbb",
"layer_id":
"6e96477fc1760c4b325af2411b0b3eeb7329ad498e1f12d3f45407b468370c87",
"base_image_id":
"e9ed59d2baf72308f3a811ebc49ff3f4e0175abf40bf636bea0160759c637999",
"cached": False
}, {
"content":
"ffdc7f85f0fe7a9b72fe172d2e54c7d39daf81d7779dcf560d729",
"layer_id":
"ccaa5ef34c2d6afacf8017f00d0ae3ce325ac9e282a49acedcbb166c8a3e23b9",
"base_image_id":
"6e96477fc1760c4b325af2411b0b3eeb7329ad498e1f12d3f45407b468370c87",
"cached": False
}],
"final_layer_id":
"55187e2caf8e5f0c8b5e6c863779701328dc9de17a3cd07525894a6e2e41339f",
"layer_index": {
"e9ed59d2baf72308f3a811ebc49ff3f4e0175abf40bf636bea0160759c637999":
{
"content": None,
"layer_id":
"e9ed59d2baf72308f3a811ebc49ff3f4e0175abf40bf636bea0160759c637999",
"base_image_id": None,
"cached": True
},
"6e96477fc1760c4b325af2411b0b3eeb7329ad498e1f12d3f45407b468370c87":
{
"content":
"730ecc32518d080377233c10f42ec832f3834cc933ff42a32cbb54bb4",
"layer_id":
"6e96477fc1760c4b325af2411b0b3eeb7329ad498e1f12d3f45407b468370c87",
"base_image_id":
"e9ed59d2baf72308f3a811ebc49ff3f4e0175abf40bf636bea0160759c637999",
"cached": False
},
"ccaa5ef34c2d6afacf8017f00d0ae3ce325ac9e282a49acedcbb166c8a3e23b9":
{
"content":
"ffdc7f85f0fe7a9b72fe172d2e54c7d39daf81d7779dcf560d729e99",
"layer_id":
"ccaa5ef34c2d6afacf8017f00d0ae3ce325ac9e282a49acedcbb166c8a3e23b9",
"base_image_id":
"6e96477fc1760c4b325af2411b0b3eeb7329ad498e1f12d3f45407b468370c87",
"cached": False
}
},
"build_container":
"ansiblefest-image-20190923-153517420660-cont",
"cache_tasks":
True,
"log_lines": [""],
"layering":
True,
"debug":
True,
"verbose":
True,
"pulled":
True,
"buildah_from_extra_args":
None,
"ansible_extra_args":
"",
"python_interpreter":
"",
"verbose_layer_names":
""
},
}
}
Path(db_path).write_text(json.dumps(db_content))
assert db.load_builds()
class Application:
def __init__(self,
debug=False,
db_path=None,
verbose=False,
init_logging=True):
"""
:param debug: bool, provide debug output if True
:param db_path: str, path to json file where the database stores the data persistently
:param verbose: bool, print verbose output
:param init_logging: bool, set up logging if True
"""
if init_logging:
self.set_logging(debug=debug, verbose=verbose)
self.verbose = verbose
self.debug = debug
self.db = Database(db_path=db_path)
self.db_path = self.db.db_root_path
@staticmethod
def set_logging(debug=False, verbose=False):
""" configure logging """
if debug:
set_logging(level=logging.DEBUG)
elif verbose:
set_logging(level=logging.INFO)
set_logging(logger_name=OUT_LOGGER,
level=logging.INFO,
format=OUT_LOGGER_FORMAT,
handler_kwargs={"stream": sys.stdout})
else:
set_logging(level=logging.WARNING)
set_logging(logger_name=OUT_LOGGER,
level=logging.INFO,
format=OUT_LOGGER_FORMAT,
handler_kwargs={"stream": sys.stdout})
def build(self, build):
"""
build container image
:param build: instance of Build
"""
if not os.path.isfile(build.playbook_path):
raise RuntimeError("No such file or directory: %s" %
build.playbook_path)
build.validate()
build.metadata.validate()
build.debug = self.debug
build.verbose = self.verbose
# we have to record as soon as possible
self.db.record_build(build)
try:
builder = self.get_builder(build)
builder.sanity_check()
# before we start messing with the base image, we need to check for its presence first
if not builder.is_base_image_present():
builder.pull()
build.pulled = True
builder.check_container_creation()
# let's record base image as a first layer
base_image_id = builder.get_image_id(build.base_image)
build.record_layer(None, base_image_id, None, cached=True)
a_runner = AnsibleRunner(build.playbook_path,
builder,
build,
debug=self.debug)
# we are about to perform the build
build.build_start_time = datetime.datetime.now()
self.db.record_build(build, build_state=BuildState.IN_PROGRESS)
if not build.python_interpreter:
build.python_interpreter = builder.find_python_interpreter()
builder.create()
except Exception:
self.db.record_build(None,
build_id=build.build_id,
build_state=BuildState.FAILED,
set_finish_time=True)
raise
try:
try:
output = a_runner.build(self.db_path)
except AbBuildUnsuccesful as ex:
b = self.db.record_build(None,
build_id=build.build_id,
build_state=BuildState.FAILED,
set_finish_time=True)
b.log_lines = ex.output.split("\n")
self.db.record_build(b)
# TODO: since this overwrites previous runs, we should likely add timestamp here
image_name = build.target_image + "-failed"
b.target_image = image_name
image_id = builder.commit(image_name)
b.final_layer_id = image_id
self.record_progress(b, None, image_id)
out_logger.info("Image build failed /o\\")
out_logger.info("The progress is saved into image '%s'",
image_name)
raise
b = self.db.record_build(None,
build_id=build.build_id,
build_state=BuildState.DONE,
set_finish_time=True)
b.log_lines = output
# commit the final image and apply all metadata
b.final_layer_id = builder.commit(build.target_image)
if not b.is_layering_on():
self.record_progress(b, None, b.final_layer_id)
else:
self.db.record_build(b)
out_logger.info("Image '%s' was built successfully \\o/",
build.target_image)
finally:
builder.clean()
def get_build(self, build_id=None):
"""
get selected build or latest build if build_id is None
:param build_id: str or None
:return: build
"""
if build_id is None:
return self.db.get_latest_build()
return self.db.get_build(build_id)
def get_logs(self, build_id=None):
"""
get logs for a specific build, if build_id is not, select the latest build
:param build_id: str or None
:return: list of str
"""
build = self.get_build(build_id=build_id)
return build.log_lines
def list_builds(self):
return self.db.load_builds()
def inspect(self, build_id=None):
"""
provide detailed information about the selected build
:param build_id: str or None
:return: dict
"""
build = self.get_build(build_id=build_id)
di = build.to_dict()
del di["log_lines"] # we have a dedicated command for that
del di["layer_index"] # internal info
return di
def push(self, target, build_id=None, force=False):
"""
push built image into a remote location, this method raises an exception when:
* the push failed or the image can't be found
* the build haven't finished yet
:param target: str, transport:details
:param build_id: id of the build or None
:param force: bool, bypass checks if True
:return: None
"""
build = self.get_build(build_id=build_id)
builder = self.get_builder(build)
builder.push(build, target, force=force)
def get_builder(self, build):
return get_builder(build.builder_name)(build, debug=self.debug)
def maybe_load_from_cache(self, content, build_id):
build = self.db.get_build(build_id)
builder = self.get_builder(build)
if not build.cache_tasks:
return
base_image_id, layer_id = self.record_progress(build, content, None)
builder.swap_working_container()
return layer_id
def get_layer(self, content, base_image_id):
"""
provide a layer for given content and base_image_id; if there
is such layer in cache store, return its layer_id
:param content:
:param base_image_id:
:return:
"""
return self.db.get_cached_layer(content, base_image_id)
def record_progress(self, build, content, layer_id, build_id=None):
"""
record build progress to the database
:param build:
:param content: str or None
:param layer_id:
:param build_id:
:return:
"""
if build_id:
build = self.db.get_build(build_id)
base_image_id = build.get_top_layer_id()
was_cached = False
if not layer_id:
# skipped task, it was cached
if content:
layer_id = self.get_layer(content, base_image_id)
builder = self.get_builder(build)
if not builder.is_image_present(layer_id):
logger.info("layer %s for content %s does not exist",
layer_id, content)
layer_id = None
if not layer_id:
return None, None
was_cached = True
build.record_layer(content, layer_id, base_image_id, cached=was_cached)
self.db.record_build(build)
return base_image_id, layer_id
def create_new_layer(self, content, build):
builder = self.get_builder(build)
timestamp = datetime.datetime.now().strftime("%Y%M%d-%H%M%S")
image_name = "%s-%s" % (build.target_image, timestamp)
# buildah doesn't accept upper case
image_name = image_name.lower()
layer_id = builder.commit(image_name, print_output=False)
base_image_id, _ = self.record_progress(build, content, layer_id)
return image_name, layer_id, base_image_id
def cache_task_result(self, content, build):
""" snapshot the container after a task was executed """
image_name, layer_id, base_image_id = self.create_new_layer(
content, build)
if not build.cache_tasks: # actually we could still cache results
return
self.db.save_layer(layer_id, base_image_id, content)
return image_name
def clean(self):
self.db.release()