def prepare_settings_for_deployment(deployment_target, output_dir,
runtime_settings):
# parse settings files
settings = collections.OrderedDict()
load_settings([
"deploy/kubernetes/shared-settings.yaml",
"deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(),
], settings)
settings.update(runtime_settings)
# make sure all keys are upper-case
settings = {key.upper(): value for key, value in settings.items()}
# minikube fix: set IMAGE_PULL_POLICY = "IfNotPresent" if running 'docker build' since it fails for other settings such as 'Always'
# https://github.com/kubernetes/minikube/issues/1395#issuecomment-296581721
# https://kubernetes.io/docs/setup/minikube/
if settings["BUILD_DOCKER_IMAGES"] and deployment_target == "minikube":
settings["IMAGE_PULL_POLICY"] = "IfNotPresent"
# set docker image tag to use when pulling images (if --build-docker-images wasn't specified) or to add to new images (if it was specified)
if runtime_settings.get("DOCKER_IMAGE_TAG"):
settings[
"DOCKER_IMAGE_TAG"] = ":" + runtime_settings["DOCKER_IMAGE_TAG"]
elif runtime_settings["BUILD_DOCKER_IMAGES"]:
settings["DOCKER_IMAGE_TAG"] = ":" + settings["TIMESTAMP"]
else:
settings["DOCKER_IMAGE_TAG"] = ":latest"
logger.info("==> Using docker image tag: %(DOCKER_IMAGE_TAG)s" % settings)
# configure deployment dir
settings["DEPLOYMENT_TEMP_DIR"] = os.path.join(
settings["DEPLOYMENT_TEMP_DIR"],
"deployments/%(TIMESTAMP)s_%(DEPLOY_TO)s" % settings)
logger.info("==> Settings:\n%s" % pformat(settings))
# re-configure logging output to write to log
log_dir = os.path.join(settings["DEPLOYMENT_TEMP_DIR"], "logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file_path = os.path.join(log_dir, "deploy.log")
sh = logging.StreamHandler(open(log_file_path, "w"))
sh.setLevel(logging.INFO)
logger.addHandler(sh)
logger.info("Starting log file: %(log_file_path)s" % locals())
# process Jinja templates to replace template variables with values from settings. Write results to temp output directory.
input_base_dir = settings["BASE_DIR"]
output_base_dir = settings["DEPLOYMENT_TEMP_DIR"]
template_file_paths = glob.glob("deploy/kubernetes/*.yaml") + \
glob.glob("deploy/kubernetes/*/*.yaml") + \
glob.glob("hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/*.yaml")
for file_path in template_file_paths:
process_jinja_template(input_base_dir, file_path, settings,
output_base_dir)
return settings
def prepare_settings_for_deployment(deployment_target, output_dir, runtime_settings):
# parse settings files
settings = collections.OrderedDict()
load_settings([
"deploy/kubernetes/shared-settings.yaml",
"deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(),
], settings)
settings.update(runtime_settings)
# make sure all keys are upper-case
settings = {key.upper(): value for key, value in settings.items()}
# minikube fix: set IMAGE_PULL_POLICY = "IfNotPresent" if running 'docker build' since it fails for other settings such as 'Always'
# https://github.com/kubernetes/minikube/issues/1395#issuecomment-296581721
# https://kubernetes.io/docs/setup/minikube/
if settings["BUILD_DOCKER_IMAGES"] and deployment_target == "minikube":
settings["IMAGE_PULL_POLICY"] = "IfNotPresent"
# set docker image tag to use when pulling images (if --build-docker-images wasn't specified) or to add to new images (if it was specified)
if runtime_settings.get("DOCKER_IMAGE_TAG"):
settings["DOCKER_IMAGE_TAG"] = ":" + runtime_settings["DOCKER_IMAGE_TAG"]
elif runtime_settings["BUILD_DOCKER_IMAGES"]:
settings["DOCKER_IMAGE_TAG"] = ":" + settings["TIMESTAMP"]
else:
settings["DOCKER_IMAGE_TAG"] = ":latest"
logger.info("==> Using docker image tag: %(DOCKER_IMAGE_TAG)s" % settings)
# configure deployment dir
settings["DEPLOYMENT_TEMP_DIR"] = os.path.join(
settings["DEPLOYMENT_TEMP_DIR"],
"deployments/%(TIMESTAMP)s_%(DEPLOY_TO)s" % settings)
logger.info("==> Settings:\n%s" % pformat(settings))
# re-configure logging output to write to log
log_dir = os.path.join(settings["DEPLOYMENT_TEMP_DIR"], "logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file_path = os.path.join(log_dir, "deploy.log")
sh = logging.StreamHandler(open(log_file_path, "w"))
sh.setLevel(logging.INFO)
logger.addHandler(sh)
logger.info("Starting log file: %(log_file_path)s" % locals())
# process Jinja templates to replace template variables with values from settings. Write results to temp output directory.
input_base_dir = settings["BASE_DIR"]
output_base_dir = settings["DEPLOYMENT_TEMP_DIR"]
template_file_paths = glob.glob("deploy/kubernetes/*.yaml") + \
glob.glob("deploy/kubernetes/*/*.yaml") + \
glob.glob("hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/*.yaml")
for file_path in template_file_paths:
process_jinja_template(input_base_dir, file_path, settings, output_base_dir)
return settings
def prepare_settings_for_deployment(deployment_target, output_dir,
runtime_settings):
# parse settings files
settings = collections.OrderedDict()
load_settings([
"deploy/kubernetes/shared-settings.yaml",
"deploy/kubernetes/%(deployment_target)s-settings.yaml" % locals(),
], settings)
settings.update(runtime_settings)
# make sure all keys are upper-case
settings = {key.upper(): value for key, value in settings.items()}
# configure deployment dir
settings["DEPLOYMENT_TEMP_DIR"] = os.path.join(
settings["DEPLOYMENT_TEMP_DIR"],
"deployments/%(TIMESTAMP)s_%(DEPLOY_TO)s" % settings)
logger.info("==> Settings:\n%s" % pformat(settings))
# re-configure logging output to write to log
log_dir = os.path.join(settings["DEPLOYMENT_TEMP_DIR"], "logs")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_file_path = os.path.join(log_dir, "deploy.log")
sh = logging.StreamHandler(open(log_file_path, "w"))
sh.setLevel(logging.INFO)
logger.addHandler(sh)
logger.info("Starting log file: %(log_file_path)s" % locals())
# process Jinja templates to replace template variables with values from settings. Write results to temp output directory.
input_base_dir = settings["BASE_DIR"]
output_base_dir = settings["DEPLOYMENT_TEMP_DIR"]
template_file_paths = glob.glob("deploy/kubernetes/*.yaml") + \
glob.glob("deploy/kubernetes/*/*.yaml") + \
glob.glob("hail_elasticsearch_pipelines/kubernetes/elasticsearch-sharded/*.yaml")
for file_path in template_file_paths:
process_jinja_template(input_base_dir, file_path, settings,
output_base_dir)
return settings