def get_content_with_env(self, template, variables):
try:
input_json = request.get_json(force=True)
for key, value in input_json.items():
if key not in EnvironmentSingleton.get_instance().get_env():
EnvironmentSingleton.get_instance().set_env_var(
str(key), str(value))
except Exception as e:
app.logger.debug(f"Exception: {e.__str__}")
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.TEMPLATE, template.strip())
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.VARIABLES, variables.strip())
try:
rendered_content = Render(
EnvironmentSingleton.get_instance().get_env_and_virtual_env(
).get(EnvConstants.TEMPLATE),
EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.VARIABLES)).rend_template()
except Exception as e:
raise ApiExceptionKubectl(
ApiCode.JINJA2_RENDER_FAILURE.value,
ErrorMessage.HTTP_CODE.get(
ApiCode.JINJA2_RENDER_FAILURE.value), e)
return Response(rendered_content, 200, mimetype="text/plain")
def rend_template(self, vars_dir=EnvironmentSingleton.get_instance().get_env_and_virtual_env().get('VARS_DIR')):
with open(vars_dir + "/" + self.variables, closefd=True) as f:
data = yaml.safe_load(f)
self.env.filters['yaml'] = self.yaml_filter
self.env.globals["environ"] = lambda key: EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(key)
self.env.globals["get_context"] = lambda: data
try:
template = self.env.get_template(self.template).render(data)
except Exception as e:
raise e
sys.stdout.write(template)
return template
def set_env(self):
input_data = request.data.decode("UTF-8", "replace").strip()
try:
env_vars_attempted = json.loads(input_data)
except Exception as e:
raise ApiExceptionDocker(
ApiCode.INVALID_JSON_PAYLOAD.value,
ErrorMessage.HTTP_CODE.get(ApiCode.INVALID_JSON_PAYLOAD.value)
% str(input_data), e)
try:
env_vars_added = EnvironmentSingleton.get_instance().set_env_vars(
env_vars_attempted)
except Exception as e:
raise ApiExceptionDocker(
ApiCode.SET_ENV_VAR_FAILURE,
ErrorMessage.HTTP_CODE.get(ApiCode.SET_ENV_VAR_FAILURE) %
str(input_data), e)
return Response(json.dumps(HttpResponse().response(
ApiCode.SUCCESS.value,
ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
env_vars_added)),
200,
mimetype="application/json")
def test_load_env_var_in_virt_env_multiple_override(self):
env = EnvironmentSingleton.get_instance()
env.set_env_vars({"ENV_TYPE": "v1"})
env_vars_set = env.set_env_vars({"ENV_TYPE": "v2"})
self.assertEqual(len(env.get_virtual_env()), 1)
self.assertEqual(env.get_virtual_env(), env_vars_set)
self.assertGreater(len(env.get_env()), 0)
def get_env_vars(self):
return Response(json.dumps(HttpResponse().response(
ApiCode.SUCCESS.value,
ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
EnvironmentSingleton.get_instance().get_env_and_virtual_env())),
200,
mimetype="application/json")
def test_load_env_var_in_virt_env_single_override(self):
env = EnvironmentSingleton.get_instance()
env.set_env_var("ENV_TYPE", "v1")
is_set = env.set_env_var("ENV_TYPE", "v2")
self.assertEqual(len(env.get_virtual_env()), 1)
self.assertEqual(env.get_virtual_env(), {"ENV_TYPE": "v2"})
self.assertEqual(is_set, True)
self.assertGreater(len(env.get_env()), 0)
def test_maxcap_for_virt_env(self):
env = EnvironmentSingleton.get_instance()
env.set_env_vars({"FOO1": "v1"})
max_cap = EnvironmentSingleton.VIRTUAL_ENV_MAX_SIZE
for i in range(0, 2 * EnvironmentSingleton.VIRTUAL_ENV_MAX_SIZE):
is_set = env.set_env_var(f"{i}", f"{i}")
self.assertEqual(is_set, False)
self.assertEqual(len(env.get_virtual_env()), EnvironmentSingleton.VIRTUAL_ENV_MAX_SIZE)
self.assertEqual(env.get_virtual_env().get(EnvironmentSingleton.VIRTUAL_ENV_MAX_SIZE), None)
self.assertGreater(len(env.get_env()), 0)
class EnvStartupSingleton:
__instance = None
__env = EnvironmentSingleton.get_instance()
@staticmethod
def get_instance():
if EnvStartupSingleton.__instance is None:
EnvStartupSingleton()
return EnvStartupSingleton.__instance
def __init__(self):
if EnvStartupSingleton.__instance is not None:
raise Exception("This class is a singleton!")
else:
EnvStartupSingleton.__instance = self
def get_config_env_vars(self):
return {
EnvConstants.APP_APPEND_LABEL:
self.__env.get_env().get(EnvConstants.APP_APPEND_LABEL).lower()
if self.__env.get_env().get(EnvConstants.APP_APPEND_LABEL) else "",
EnvConstants.ENV_EXPIRE_IN:
int(self.__env.get_env().get(EnvConstants.ENV_EXPIRE_IN))
if self.__env.get_env().get(EnvConstants.ENV_EXPIRE_IN) else 1440,
EnvConstants.SCHEDULER_POLL_INTERVAL:
int(self.__env.get_env().get(EnvConstants.SCHEDULER_POLL_INTERVAL))
if self.__env.get_env().get(
EnvConstants.SCHEDULER_POLL_INTERVAL) else 1200,
EnvConstants.APP_IP_PORT:
self.__env.get_env().get(EnvConstants.APP_IP_PORT).strip().lower()
if self.__env.get_env().get(EnvConstants.APP_IP_PORT) else None,
EnvConstants.PORT:
int(self.__env.get_env().get(EnvConstants.PORT).strip()) if
self.__env.get_env().get(EnvConstants.PORT) is not None else 8080,
EnvConstants.EUREKA_SERVER:
self.__env.get_env().get(EnvConstants.EUREKA_SERVER).strip()
if self.__env.get_env().get(EnvConstants.EUREKA_SERVER) else None,
EnvConstants.FLUENTD_IP_PORT:
self.__env.get_env().get(EnvConstants.FLUENTD_IP_PORT).strip() if
self.__env.get_env().get(EnvConstants.FLUENTD_IP_PORT) else None,
EnvConstants.HTTP_AUTH_TOKEN:
self.__env.get_env().get(EnvConstants.HTTP_AUTH_TOKEN).strip() if
self.__env.get_env().get(EnvConstants.HTTP_AUTH_TOKEN) else "None",
EnvConstants.HTTPS_ENABLE:
bool(self.__env.get_env().get(EnvConstants.HTTPS_ENABLE).strip())
if self.__env.get_env().get(EnvConstants.HTTPS_ENABLE) else False,
EnvConstants.HTTPS_CERT:
self.__env.get_env().get(EnvConstants.HTTPS_CERT).strip()
if self.__env.get_env().get(
EnvConstants.HTTPS_CERT) else "https/cert.pem",
EnvConstants.HTTPS_KEY:
self.__env.get_env().get(EnvConstants.HTTPS_KEY).strip()
if self.__env.get_env().get(
EnvConstants.HTTPS_KEY) else "https/key.pem"
}
def __init__(self, template=None, variables=None):
"""
Custom jinja2 render
"""
self.template = template
self.variables = variables
self.env = jinja2.Environment(
loader=jinja2.FileSystemLoader(
EnvironmentSingleton.get_instance().get_env_and_virtual_env().get('TEMPLATES_DIR')),
extensions=['jinja2.ext.autoescape', 'jinja2.ext.do', 'jinja2.ext.loopcontrols', 'jinja2.ext.with_'],
autoescape=True,
trim_blocks=True)
class CmdUtils:
__env = EnvironmentSingleton.get_instance()
@staticmethod
def run_cmd_detached(command):
p = subprocess.Popen(command,
stdout=None,
stderr=None,
shell=True,
env=CmdUtils.__env.get_env_and_virtual_env())
print("Opened pid {} for command {}".format(p.pid, command))
@staticmethod
def run_cmd_shell_true(command):
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=CmdUtils.__env.get_env_and_virtual_env(),
shell=True)
return CmdUtils.__get_subprocess_data(p)
@staticmethod
def run_cmd_shell_false(command):
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=CmdUtils.__env.get_env_and_virtual_env())
return CmdUtils.__get_subprocess_data(p)
@staticmethod
def __get_subprocess_data(p):
lines_to_slice = 1000
[out, err] = p.communicate()
return {
"out":
"\n".join(
out.decode("UTF-8",
"replace").split("\n")[-lines_to_slice:]).rstrip(),
"err":
"\n".join(
err.decode("UTF-8",
"replace").split("\n")[-lines_to_slice:]).rstrip(),
"code":
p.returncode,
"pid":
p.pid,
"args":
p.args
}
class EnvStartupSingleton:
__instance = None
__env = EnvironmentSingleton.get_instance()
@staticmethod
def get_instance():
if EnvStartupSingleton.__instance is None:
EnvStartupSingleton()
return EnvStartupSingleton.__instance
def __init__(self):
if EnvStartupSingleton.__instance is not None:
raise Exception("This class is a singleton!")
else:
EnvStartupSingleton.__instance = self
def get_config_env_vars(self):
return {
EnvConstants.APP_IP: self.__env.get_env_and_virtual_env().get(
EnvConstants.APP_IP).strip().lower() if self.__env.get_env_and_virtual_env().get(
EnvConstants.APP_IP) else "localhost",
EnvConstants.PORT: int(self.__env.get_env_and_virtual_env().get(
EnvConstants.PORT).strip()) if self.__env.get_env_and_virtual_env().get(
EnvConstants.PORT) is not None else 8080,
EnvConstants.EUREKA_SERVER: self.__env.get_env_and_virtual_env().get(
EnvConstants.EUREKA_SERVER).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.EUREKA_SERVER) else None,
EnvConstants.FLUENTD_IP_PORT: self.__env.get_env_and_virtual_env().get(
EnvConstants.FLUENTD_IP_PORT).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.FLUENTD_IP_PORT) else None,
EnvConstants.HTTP_AUTH_USER: self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTP_AUTH_USER).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTP_AUTH_USER) else "admin",
EnvConstants.HTTP_AUTH_PASSWORD: self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTP_AUTH_PASSWORD).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTP_AUTH_PASSWORD) else "estuaryoss123!",
EnvConstants.HTTPS_ENABLE: bool(self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_ENABLE).strip()) if self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_ENABLE) else False,
EnvConstants.HTTPS_CERT: self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_CERT).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_CERT) else "https/cert.pem",
EnvConstants.HTTPS_KEY: self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_KEY).strip() if self.__env.get_env_and_virtual_env().get(
EnvConstants.HTTPS_KEY) else "https/key.pem"
}
def test_load_env_empty_virt_env(self):
env = EnvironmentSingleton.get_instance()
self.assertGreater(len(env.get_env()), 0)
def test_load_env_var_in_virt_env_empty_key_in_dict(self):
env = EnvironmentSingleton.get_instance()
env_vars_set = env.set_env_vars({"": "v2"})
self.assertEqual(env_vars_set, {})
self.assertGreater(len(env.get_env()), 0)
class EnvInit:
# env constants
init = {}
WORKSPACE = "tmp"
init[EnvConstants.WORKSPACE] = WORKSPACE
app = AppCreatorSingleton.get_instance().get_app()
if EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.WORKSPACE):
init[EnvConstants.WORKSPACE] = EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(EnvConstants.WORKSPACE)
# take paths from env if they exists
init[EnvConstants.TEMPLATES_DIR] = EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(
EnvConstants.TEMPLATES_DIR) if EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(
EnvConstants.TEMPLATES_DIR) else init.get(
EnvConstants.WORKSPACE) + "/templates"
init[EnvConstants.DEPLOY_WITH] = EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(
EnvConstants.DEPLOY_WITH) if EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(EnvConstants.DEPLOY_WITH) else "docker"
init[EnvConstants.VARS_DIR] = EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(
EnvConstants.VARS_DIR) if EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(EnvConstants.VARS_DIR) else init.get(
EnvConstants.WORKSPACE) + "/variables"
init[EnvConstants.MAX_DEPLOYMENTS] = int(
EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.MAX_DEPLOYMENTS)
) if EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.MAX_DEPLOYMENTS) else 10
init[EnvConstants.DEPLOY_PATH] = init.get(
EnvConstants.WORKSPACE) + "/deployments"
if not EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.VARS_DIR):
EnvironmentSingleton.get_instance().get_env_and_virtual_env()[
EnvConstants.VARS_DIR] = init.get(EnvConstants.VARS_DIR)
app.logger.debug(
f"{EnvConstants.VARS_DIR} env var not set, defaulting to : " +
str(EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.VARS_DIR)))
if not EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.TEMPLATES_DIR):
EnvironmentSingleton.get_instance().get_env_and_virtual_env()[
EnvConstants.TEMPLATES_DIR] = init.get(EnvConstants.TEMPLATES_DIR)
app.logger.debug(
f"{EnvConstants.TEMPLATES_DIR} env var not set, defaulting to : " +
str(EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.TEMPLATES_DIR)))
app.logger.debug("WORKSPACE : " + init.get(EnvConstants.WORKSPACE))
app.logger.debug("DEPLOY_PATH : " + init.get(EnvConstants.DEPLOY_PATH))
app.logger.debug("DEPLOY_WITH : " + init.get(EnvConstants.DEPLOY_WITH))
app.logger.debug("MAX_DEPLOYMENTS : " +
str(init.get(EnvConstants.MAX_DEPLOYMENTS)))
app.logger.debug("TEMPLATES_DIR : " + init.get(EnvConstants.TEMPLATES_DIR))
app.logger.debug("VARS_DIR : " + init.get(EnvConstants.VARS_DIR))
def yaml_filter(self, value):
return yaml.dump(value, Dumper=yaml.RoundTripDumper, indent=4)
def env_override(self, value, key):
return os.getenv(key, value)
def rend_template(self, vars_dir=EnvironmentSingleton.get_instance().get_env_and_virtual_env().get('VARS_DIR')):
with open(vars_dir + "/" + self.variables, closefd=True) as f:
data = yaml.safe_load(f)
self.env.filters['yaml'] = self.yaml_filter
self.env.globals["environ"] = lambda key: EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(key)
self.env.globals["get_context"] = lambda: data
try:
template = self.env.get_template(self.template).render(data)
except Exception as e:
raise e
sys.stdout.write(template)
return template
def get_jinja2env(self):
return self.env
if __name__ == '__main__':
render = Render(EnvironmentSingleton.get_instance().get_env_and_virtual_env().get('TEMPLATE'),
EnvironmentSingleton.get_instance().get_env_and_virtual_env().get('VARIABLES')).rend_template()
from rest.utils.thread_utils import ThreadUtils
app = AppCreatorSingleton.get_instance().get_app()
auth = HTTPBasicAuth()
logger = \
sender.FluentSender(tag=properties.get('name'),
host=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.FLUENTD_IP_PORT).split(":")[0],
port=int(EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.FLUENTD_IP_PORT).split(":")[1])) \
if EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.FLUENTD_IP_PORT) else None
fluentd_service = Fluentd(logger)
message_dumper = MessageDumper()
env = EnvironmentSingleton.get_instance()
users = {
EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.HTTP_AUTH_USER):
generate_password_hash(
EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.HTTP_AUTH_PASSWORD))
}
@auth.verify_password
def verify_password(username, password):
if username in users and \
check_password_hash(users.get(username), password):
return username
return os.getenv(key, value)
def rend_template(self, vars_dir=EnvInit.init.get(EnvConstants.VARS_DIR)):
with open(vars_dir + "/" + self.variables, closefd=True) as f:
data = yaml.safe_load(f)
self.env.filters['yaml'] = self.yaml_filter
self.env.globals[
"environ"] = lambda key: EnvironmentSingleton.get_instance(
).get_env_and_virtual_env().get(key)
self.env.globals["get_context"] = lambda: data
try:
template = self.env.get_template(self.template).render(data)
except Exception as e:
raise e
sys.stdout.write(template)
return template
def get_jinja2env(self):
return self.env
if __name__ == '__main__':
render = Render(
EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.TEMPLATE),
EnvironmentSingleton.get_instance().get_env_and_virtual_env().get(
EnvConstants.VARIABLES)).rend_template()
def start_deployment(self):
docker_utils = DockerUtils()
token = token_hex(8)
deployment_id = request.headers.get("Deployment-Id").lower(
) if request.headers.get("Deployment-Id") else token
deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{deployment_id}"
file = f"{deploy_dir}/docker-compose.yml"
header_key = 'Eureka-Server'
eureka_server_header = request.headers.get(f"{header_key}")
config_env_vars = EnvStartupSingleton.get_instance(
).get_config_env_vars()
input_data = request.data.decode('UTF-8').strip()
status = CmdUtils.run_cmd_shell_false(["docker", "ps"])
if "Cannot connect to the Docker daemon".lower() in status.get(
'err').lower():
raise ApiExceptionDocker(
ApiCode.DOCKER_DAEMON_NOT_RUNNING.value,
ErrorMessage.HTTP_CODE.get(
ApiCode.DOCKER_DAEMON_NOT_RUNNING.value),
status.get('err'))
active_deployments = docker_utils.get_active_deployments()
if len(active_deployments) >= EnvInit.init.get(
EnvConstants.MAX_DEPLOYMENTS):
raise ApiExceptionDocker(
ApiCode.MAX_DEPLOYMENTS_REACHED.value,
ErrorMessage.HTTP_CODE.get(
ApiCode.MAX_DEPLOYMENTS_REACHED.value) %
str(EnvInit.init.get(EnvConstants.MAX_DEPLOYMENTS)),
active_deployments)
try:
template_file_name = f"deployment_{deployment_id}.yml"
template_file_path = f"{EnvInit.init.get(EnvConstants.TEMPLATES_DIR)}/{template_file_name}"
app.logger.debug({
"msg": {
"file": template_file_path,
"file_content": f"{input_data}"
}
})
IOUtils.write_to_file(template_file_path, input_data)
IOUtils.create_dir(deploy_dir)
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.TEMPLATE, template_file_name)
env_vars = EnvironmentSingleton.get_instance(
).get_env_and_virtual_env()
render = Render(env_vars.get(EnvConstants.TEMPLATE),
env_vars.get(EnvConstants.VARIABLES))
if config_env_vars.get(
EnvConstants.EUREKA_SERVER) and config_env_vars.get(
EnvConstants.APP_IP_PORT):
# if {{app_ip_port}} and {{eureka_server}} then register that instance too
if '{{app_ip_port}}' in input_data and '{{eureka_server}}' in input_data:
eureka_server = config_env_vars.get(
EnvConstants.EUREKA_SERVER)
# header value overwrite the eureka server
if eureka_server_header:
eureka_server = eureka_server_header
input_data = render.get_jinja2env().get_template(
env_vars.get(EnvConstants.TEMPLATE)).render({
"deployment_id":
f"{deployment_id}",
"eureka_server":
eureka_server,
"app_ip_port":
config_env_vars.get(
EnvConstants.APP_IP_PORT).split("/")[0]
})
os.remove(template_file_path) if os.path.exists(
template_file_path) else None
app.logger.debug(
{"msg": {
"file": file,
"file_content": f"{input_data}"
}})
IOUtils.write_to_file(file, input_data) if input_data else None
CmdUtils.run_cmd_detached(
rf'''docker-compose -f {file} pull && docker-compose -f {file} up -d'''
)
except Exception as e:
app.logger.debug({"msg": docker_utils.down(file)})
raise ApiExceptionDocker(
ApiCode.DEPLOY_START_FAILURE.value,
ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
e)
DeploymentMetadataSingleton.get_instance() \
.delete_metadata_for_inactive_deployments(DockerUtils.get_active_deployments())
metadata = DeploymentReader.get_metadata_for_deployment(
IOUtils.read_file(file=file))
IOUtils.write_to_file_dict(f"{deploy_dir}/metadata.json", metadata)
DeploymentMetadataSingleton.get_instance().set_metadata_for_deployment(
deployment_id, metadata)
return Response(json.dumps(HttpResponse().response(
ApiCode.SUCCESS.value,
ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value), deployment_id)),
200,
mimetype="application/json")
io_utils.create_dirs([Path(EnvInit.init.get(EnvConstants.DEPLOY_PATH)),
Path(EnvInit.init.get(EnvConstants.TEMPLATES_DIR)),
Path(EnvInit.init.get(EnvConstants.VARS_DIR))])
DockerEnvExpireScheduler(fluentd_utils=DockerView.fluentd,
poll_interval=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.SCHEDULER_POLL_INTERVAL), # seconds
env_expire_in=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.ENV_EXPIRE_IN)).start() # minutes
KubectlEnvExpireScheduler(fluentd_utils=KubectlView.fluentd,
poll_interval=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.SCHEDULER_POLL_INTERVAL),
env_expire_in=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.ENV_EXPIRE_IN)).start()
environ_dump = message_dumper.dump_message(EnvironmentSingleton.get_instance().get_env_and_virtual_env())
ip_port_dump = message_dumper.dump_message(
{"host": host, "port": EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.PORT)})
app.logger.debug({"msg": environ_dump})
app.logger.debug({"msg": ip_port_dump})
app.logger.debug({"msg": EnvStartupSingleton.get_instance().get_config_env_vars()})
logger = \
sender.FluentSender(tag=properties.get('name'),
host=EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.FLUENTD_IP_PORT).split(":")[0],
port=int(EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.FLUENTD_IP_PORT).split(":")[1])) \
if EnvStartupSingleton.get_instance().get_config_env_vars().get(EnvConstants.FLUENTD_IP_PORT) else None
fluentd_utils = Fluentd(logger)
def start_deployment_with_template(self, template, variables):
http = HttpResponse()
kubectl_utils = KubectlUtils()
fluentd_tag = "deploy_start"
try:
input_json = request.get_json(force=True)
for key, value in input_json.items():
if key not in EnvironmentSingleton.get_instance().get_env():
EnvironmentSingleton.get_instance().set_env_var(
str(key), str(value))
except Exception as e:
app.logger.debug(f"Exception: {e.__str__()}")
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.TEMPLATE, template.strip())
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.VARIABLES, variables.strip())
app.logger.debug({
"msg": {
"template_file":
EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.TEMPLATE)
}
})
app.logger.debug({
"msg": {
"variables_file":
EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.VARIABLES)
}
})
token = token_hex(8)
deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{token}"
file = f"{deploy_dir}/k8s-deployment.yml"
try:
r = Render(
EnvironmentSingleton.get_instance().get_env_and_virtual_env(
).get(EnvConstants.TEMPLATE),
EnvironmentSingleton.get_instance().get_env_and_virtual_env().
get(EnvConstants.VARIABLES))
IOUtils.create_dir(deploy_dir)
IOUtils.write_to_file(file)
IOUtils.write_to_file(file, r.rend_template())
status = kubectl_utils.up(f"{file}")
self.fluentd.emit(tag=fluentd_tag, msg={"msg": status})
if status.get('err'):
raise Exception(status.get('error'))
result = str(token)
except Exception as e:
raise ApiExceptionKubectl(
ApiCode.DEPLOY_START_FAILURE.value,
ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
e)
return Response(json.dumps(
http.response(ApiCode.SUCCESS.value,
ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
result)),
200,
mimetype="application/json")
EnvConstants.PORT)
message_dumper = MessageDumper()
host = '0.0.0.0'
fluentd_tag = "startup"
if EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.EUREKA_SERVER):
EurekaRegistrator(EnvStartupSingleton.get_instance(
).get_config_env_vars().get(EnvConstants.EUREKA_SERVER)).register_app(
EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.APP_IP),
EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.PORT))
environ_dump = message_dumper.dump_message(
EnvironmentSingleton.get_instance().get_env_and_virtual_env())
ip_port_dump = message_dumper.dump_message({"host": host, "port": port})
app.logger.debug({"msg": environ_dump})
app.logger.debug({"msg": ip_port_dump})
app.logger.debug(
{"msg": EnvStartupSingleton.get_instance().get_config_env_vars()})
fluentd_service.emit(tag=fluentd_tag, msg=environ_dump)
is_https = EnvStartupSingleton.get_instance().get_config_env_vars().get(
EnvConstants.HTTPS_ENABLE)
https_cert_path = EnvStartupSingleton.get_instance().get_config_env_vars(
).get(EnvConstants.HTTPS_CERT)
https_prv_key_path = EnvStartupSingleton.get_instance(
).get_config_env_vars().get(EnvConstants.HTTPS_KEY)
def test_load_env_var_in_virt_env_single(self):
env = EnvironmentSingleton.get_instance()
is_set = env.set_env_var("FOO1", "v1")
self.assertEqual(len(env.get_virtual_env()), 1)
self.assertEqual(is_set, True)
self.assertGreater(len(env.get_env()), 0)
def start_deployment_with_templates(self, template, variables):
http = HttpResponse()
docker_utils = DockerUtils()
token = token_hex(8)
deployment_id = request.headers.get("Deployment-Id").lower(
) if request.headers.get("Deployment-Id") else token
deploy_dir = f"{EnvInit.init.get(EnvConstants.DEPLOY_PATH)}/{deployment_id}"
file = f"{deploy_dir}/docker-compose.yml"
try:
input_json = request.get_json(force=True)
for key, value in input_json.items():
if key not in EnvironmentSingleton.get_instance().get_env():
EnvironmentSingleton.get_instance().set_env_var(
str(key), str(value))
except Exception as e:
app.logger.debug(
f"Could not parse the input from the request as JSON: {e.__str__()}"
)
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.TEMPLATE, template.strip())
EnvironmentSingleton.get_instance().set_env_var(
EnvConstants.VARIABLES, variables.strip())
env_vars = EnvironmentSingleton.get_instance().get_env_and_virtual_env(
)
app.logger.debug(
{"msg": {
"template_file": env_vars.get(EnvConstants.TEMPLATE)
}})
app.logger.debug(
{"msg": {
"variables_file": env_vars.get(EnvConstants.VARIABLES)
}})
status = CmdUtils.run_cmd_shell_false(["docker", "ps"])
if "Cannot connect to the Docker daemon".lower() in status.get(
'err').lower():
raise ApiExceptionDocker(
ApiCode.DOCKER_DAEMON_NOT_RUNNING.value,
ErrorMessage.HTTP_CODE.get(
ApiCode.DOCKER_DAEMON_NOT_RUNNING.value),
status.get('err'))
active_deployments = docker_utils.get_active_deployments()
if len(active_deployments) >= EnvInit.init.get(
EnvConstants.MAX_DEPLOYMENTS):
raise ApiExceptionDocker(
ApiCode.MAX_DEPLOYMENTS_REACHED.value,
ErrorMessage.HTTP_CODE.get(
ApiCode.MAX_DEPLOYMENTS_REACHED.value) %
str(EnvInit.init.get(EnvConstants.MAX_DEPLOYMENTS)),
active_deployments)
try:
r = Render(env_vars.get(EnvConstants.TEMPLATE),
env_vars.get(EnvConstants.VARIABLES))
IOUtils.create_dir(deploy_dir)
IOUtils.write_to_file(file, r.rend_template())
CmdUtils.run_cmd_detached(
rf'''docker-compose -f {file} pull && docker-compose -f {file} up -d'''
)
except Exception as e:
raise ApiExceptionDocker(
ApiCode.DEPLOY_START_FAILURE.value,
ErrorMessage.HTTP_CODE.get(ApiCode.DEPLOY_START_FAILURE.value),
e)
DeploymentMetadataSingleton.get_instance() \
.delete_metadata_for_inactive_deployments(DockerUtils.get_active_deployments())
metadata = DeploymentReader.get_metadata_for_deployment(
IOUtils.read_file(file=file))
IOUtils.write_to_file_dict(f"{deploy_dir}/metadata.json", metadata)
DeploymentMetadataSingleton.get_instance().set_metadata_for_deployment(
deployment_id, metadata)
return Response(json.dumps(
http.response(ApiCode.SUCCESS.value,
ErrorMessage.HTTP_CODE.get(ApiCode.SUCCESS.value),
deployment_id)),
200,
mimetype="application/json")
def test_load_env_var_in_virt_env_single_empty_key(self):
env = EnvironmentSingleton.get_instance()
is_set = env.set_env_var("", "v1")
self.assertEqual(is_set, False)
self.assertGreater(len(env.get_env()), 0)