def create(self, spec):
app_path = os.path.join(self.apps_dir, spec["name"])
make_dir(app_path, clean=True)
with open(os.path.join(app_path, "spec.json"), "w+") as spec_file:
spec_file.write(json.dumps(spec))
m = self._build_meta(spec, app_path)
return m
def _configure_app_loggers(self):
make_dir(LogSettings.APPS_DIRECTORY)
apps = App.get_app()
for app in apps:
if not app in self._app_loggers:
logger = self._make_app_logger(app.name)
self._app_loggers[app.name] = logger
def create(self, spec):
app_path = os.path.join(self.apps_dir, spec["name"])
make_dir(app_path, clean=True)
with open(os.path.join(app_path, "spec.json"), "w+") as spec_file:
spec_file.write(json.dumps(spec))
m = self._build_meta(spec, app_path)
return m
def build(self):
# only initiate the build if the current spec is different than the last spec built
if self._json != self.last_build:
try:
# clean up the old build
build_path = os.path.join(self.path, "build")
make_dir(build_path, clean=True)
# copy new file and replace all template placeholders with parameters
build_dirs = ["components", "deployments", "images"]
for bd in build_dirs:
bd_path = os.path.join(build_path, bd)
shutil.copytree(os.path.join(self.path, bd), bd_path)
for root, dirs, files in os.walk(
os.path.join(build_path, bd)):
for f in files:
fill_template(os.path.join(root, f),
self.parameters)
# now that all templates are filled, build/upload images
for image in self.images:
try:
image_name = MainSettings.REGISTRY_NAME + "/" + self.full_name + "-" + image[
"name"]
image_path = os.path.join(build_path, "images",
image["name"])
subprocess.check_call(
['docker', 'build', '-t', image_name, image_path])
info_log(
self.TAG,
"Squashing and pushing {} to private registry...".
format(image_name))
subprocess.check_call([
os.path.join(MainSettings.ROOT, "util",
"squash-and-push"), image_name
])
except subprocess.CalledProcessError as e:
raise Service.BuildFailedException(
"could not build service {0}: {1}".format(
self.full_name, e))
except Service.BuildFailedException as e:
error_log(
self.TAG, "could not build service: {0}: {1}".format(
self.full_name, e))
return False
info_log(self.TAG, "Successfully built {0}".format(self.full_name))
# write latest build parameters
self.index.save_service(self)
return True
else:
info_log(
self.TAG,
"Image {0} not changed since last build. Not rebuilding.".
format(self.full_name))
return True
def _configure_root_logger(self):
make_dir(LogSettings.ROOT_DIRECTORY)
log_dir = os.path.join(LogSettings.ROOT_DIRECTORY, "root")
make_dir(log_dir)
logging.basicConfig(format=LogWriter.ROOT_FORMAT)
self._root_logger = logging.getLogger(__name__)
self._root_logger.propagate = False
self._set_logging_config(log_dir, LogSettings.ROOT_FILE, self._root_logger)
def __init__(self, queue, preload):
super(Builder, self).__init__()
self._build_queue = queue
self._pool = NoDaemonPool(processes=NUM_WORKERS)
multiprocessing.freeze_support()
self._stopped = False
self._preload = preload
make_dir(LOG_DIR)
def __init__(self, queue, preload):
super(Builder, self).__init__()
self._build_queue = queue
self._pool = NoDaemonPool(processes=NUM_WORKERS)
multiprocessing.freeze_support()
self._stopped = False
self._preload = preload
make_dir(LOG_DIR)
def _configure_root_logger(self):
make_dir(LogSettings.ROOT_DIRECTORY)
log_dir = os.path.join(LogSettings.ROOT_DIRECTORY, "root")
make_dir(log_dir)
logging.basicConfig(format=LogWriter.ROOT_FORMAT)
self._root_logger = logging.getLogger(__name__)
self._root_logger.propagate = False
self._set_logging_config(log_dir, LogSettings.ROOT_FILE,
self._root_logger)
def build(self, build_base=False, preload=False):
try:
# clean up the old build and record the start of a new build
build_path = os.path.join(self.path, "build")
make_dir(build_path, clean=True)
App.index.update_build_state(self, App.BuildState.BUILDING)
App.index.update_last_build_time(self)
# fetch the repo
self._fetch_repo()
# ensure that the service dependencies are all build
info_log(self.TAG,
"Building service dependencies...",
app=self.name)
for service in self.services:
built_service = service.build()
if not built_service:
raise App.BuildFailedException(
"could not build service {}".format(service.full_name))
# copy new file and replace all template placeholders with parameters
self._fill_templates(build_path)
if build_base:
self._build_base_image()
app_img_path = os.path.join(build_path, "app")
make_dir(app_img_path)
shutil.copytree(self.repo, os.path.join(app_img_path, "repo"))
if "dockerfile" in self.dependencies:
self._build_with_dockerfile(build_path)
else:
self._build_without_dockerfile(build_path)
# push the app image to the private registry
self._push_image()
# if preload is set, send the app image to all nodes
if preload:
self._preload_image()
except App.BuildFailedException as e:
App.index.update_build_state(self, App.BuildState.FAILED)
error_log(self.TAG, str(e), app=self.name)
return
info_log(self.TAG,
"Successfully built app {0}".format(self.name),
app=self.name)
App.index.update_build_state(self, App.BuildState.COMPLETED)
def build(self, build_base=False, preload=False):
try:
# clean up the old build and record the start of a new build
build_path = os.path.join(self.path, "build")
make_dir(build_path, clean=True)
App.index.update_build_state(self, App.BuildState.BUILDING)
App.index.update_last_build_time(self)
# fetch the repo
self._fetch_repo()
# ensure that the service dependencies are all build
info_log(self.TAG, "Building service dependencies...", app=self.name)
for service in self.services:
built_service = service.build()
if not built_service:
raise App.BuildFailedException("could not build service {}".format(service.full_name))
# copy new file and replace all template placeholders with parameters
self._fill_templates(build_path)
if build_base:
self._build_base_image()
app_img_path = os.path.join(build_path, "app")
make_dir(app_img_path)
shutil.copytree(self.repo, os.path.join(app_img_path, "repo"))
if "dockerfile" in self.dependencies:
self._build_with_dockerfile(build_path)
else:
self._build_without_dockerfile(build_path)
# push the app image to the private registry
self._push_image()
# if preload is set, send the app image to all nodes
if preload:
self._preload_image()
except App.BuildFailedException as e:
App.index.update_build_state(self, App.BuildState.FAILED)
error_log(self.TAG, str(e), app=self.name)
return
info_log(self.TAG, "Successfully built app {0}".format(self.name), app=self.name)
App.index.update_build_state(self, App.BuildState.COMPLETED)
def _configure_root_loggers(self):
make_dir(LogSettings.ROOT_DIRECTORY)
log_dir = os.path.join(LogSettings.ROOT_DIRECTORY, "root")
make_dir(log_dir)
logging.basicConfig(format=LogWriter.ROOT_FORMAT)
# zeromq publishing logger configuration
self._stream_logger = logging.getLogger(__name__ + "-stream")
self._stream_logger.setLevel(LogSettings.LEVEL)
ph = LogWriter.PublisherHandler()
ph.setFormatter(Formatter(LogWriter.BINDER_FORMAT))
self._stream_logger.addHandler(ph)
# root logger configuration
self._root_logger = logging.getLogger(__name__)
self._root_logger.propagate = False
self._set_logging_config(log_dir, LogSettings.ROOT_FILE, self._root_logger)
def _configure_root_loggers(self):
make_dir(LogSettings.ROOT_DIRECTORY)
log_dir = os.path.join(LogSettings.ROOT_DIRECTORY, "root")
make_dir(log_dir)
logging.basicConfig(format=LogWriter.ROOT_FORMAT)
# zeromq publishing logger configuration
self._stream_logger = logging.getLogger(__name__ + '-stream')
self._stream_logger.setLevel(LogSettings.LEVEL)
ph = LogWriter.PublisherHandler()
ph.setFormatter(Formatter(LogWriter.BINDER_FORMAT))
self._stream_logger.addHandler(ph)
# root logger configuration
self._root_logger = logging.getLogger(__name__)
self._root_logger.propagate = False
self._set_logging_config(log_dir, LogSettings.ROOT_FILE, self._root_logger)
def build(self):
# only initiate the build if the current spec is different than the last spec built
if self._json != self.last_build:
try:
# clean up the old build
build_path = os.path.join(self.path, "build")
make_dir(build_path, clean=True)
# copy new file and replace all template placeholders with parameters
build_dirs = ["components", "deployments", "images"]
for bd in build_dirs:
bd_path = os.path.join(build_path, bd)
shutil.copytree(os.path.join(self.path, bd), bd_path)
for root, dirs, files in os.walk(os.path.join(build_path, bd)):
for f in files:
fill_template(os.path.join(root, f), self.parameters)
# now that all templates are filled, build/upload images
for image in self.images:
try:
image_name = REGISTRY_NAME + "/" + self.full_name + "-" + image["name"]
image_path = os.path.join(build_path, "images", image["name"])
subprocess.check_call(['docker', 'build', '-t', image_name, image_path])
print("Squashing and pushing {} to private registry...".format(image_name))
subprocess.check_call([os.path.join(ROOT, "util", "squash-and-push"), image_name])
except subprocess.CalledProcessError as e:
raise Service.BuildFailedException("could not build service {0}: {1}".format(self.full_name, e))
except Service.BuildFailedException as e:
print("could not build service: {0}: {1}".format(self.full_name, e))
return False
print("Successfully built {0}".format(self.full_name))
# write latest build parameters
self.index.save_service(self)
return True
else:
print("Image {0} not changed since last build. Not rebuilding.".format(self.full_name))
return True
def make_app_path(self, app):
path = self.get_app_path(app)
make_dir(path)
return path
def __init__(self, root):
self.apps_dir = os.path.join(root, FileAppIndex.APPS_DIR)
make_dir(self.apps_dir)
def _configure_app_loggers(self):
make_dir(LogSettings.APPS_DIRECTORY)
apps = App.get_app()
for app in apps:
if not app.name in self._app_loggers:
self._make_app_loggers(app.name)
def make_app_path(self, app):
path = self.get_app_path(app)
make_dir(path)
return path
def __init__(self, root):
self.apps_dir = os.path.join(root, FileAppIndex.APPS_DIR)
make_dir(self.apps_dir)