def deploy_application(self, app_id, app_info):
fmlogger.debug("Deploying application %s" % app_info['app_name'])
self._copy_creds(app_info)
app_data = {}
app_data['status'] = 'setting-up-kubernetes-config'
app_db.App().update(app_id, app_data)
try:
self._setup_kube_config(app_info)
except Exception as e:
fmlogger.error(
"Exception encountered in obtaining kube config %s" % e)
app_db.App().update(app_id, {'status': str(e)})
# Resolve environment
common_functions.resolve_environment_multicont(app_id, app_info)
app_details = {}
app_data = {}
app_data['status'] = 'deploying'
app_db.App().update(app_id, app_data)
try:
self._deploy(app_id, app_info)
fmlogger.debug("Done deploying application %s" %
app_info['app_name'])
except exceptions.AppDeploymentFailure as e:
fmlogger.error(str(e))
app_data['status'] = 'deployment-failed ' + str(
e) + " " + e.get_message()
app_db.App().update(app_id, app_data)
def test_app_insert(self):
app_data = {}
app_data['name'] = self._get_app_name()
app_data['location'] = 'abc'
app_data['version'] = 'version'
app_data['dep_target'] = 'local'
app_data['env_id'] = 1
new_app_id = app.App().insert(app_data)
self.assertIsNotNone(new_app_id, "App not inserted properly")
app.App().delete(new_app_id)
def test_app_delete(self):
app_data = {}
app_data['name'] = self._get_app_name()
app_data['location'] = 'abc'
app_data['version'] = 'version'
app_data['dep_target'] = 'local'
app_data['env_id'] = 1
new_app_id = app.App().insert(app_data)
app.App().delete(new_app_id)
deleted_app = app.App().get(new_app_id)
self.assertIsNone(deleted_app)
def delete_application(self, app_id, app_info):
fmlogger.debug("Deleting application %s" % app_info['app_name'])
app_obj = app_db.App().get(app_id)
try:
app_output_config = ast.literal_eval(app_obj.output_config)
self._delete_service(app_info, app_output_config['service_name'])
pod_name_list = app_output_config['pod_name']
for pod in pod_name_list:
self._delete_pod(app_info, pod)
except Exception as e:
fmlogger.error(e)
app_db.App().delete(app_id)
fmlogger.debug("Done deleting application %s" % app_info['app_name'])
def is_app_ready(app_url, app_id='', timeout=300):
ready = False
count = 0
num_of_oks = 10
oks = 0
while count < timeout and not ready:
try:
response = requests.get(app_url)
if response.status_code == 200 or response.status_code == 404:
oks = oks + 1
if oks == num_of_oks:
ready = True
break
except Exception as e:
fmlogging.error(e)
count = count + 1
time.sleep(3)
# After every 10 counts check if app still exists
if app_id:
if count % 10 == 0:
app_obj = app_db.App().get(app_id)
if not app_obj:
count = timeout
return ready
def delete_application(self, app_id, app_info):
fmlogger.debug("Deleting application %s %s" % (app_id, app_info['app_name']))
try:
app_db.App().update(app_id, {'status': constants.DELETING_APP})
app_obj = app_db.App().get(app_id)
output_config = ast.literal_eval(app_obj.output_config)
cont_id = output_config['cont_id'].strip()
err, output = self.docker_handler.stop_container(cont_id)
if err:
fmlogger.debug("Encountered error in stopping container %s." % cont_id)
self.docker_handler.remove_container(cont_id)
except Exception as e:
fmlogger.error(e)
app_db.App().delete(app_id)
fmlogger.debug("Done deleting application")
def _deploy_pod(self, app_id, app_info):
df_file = self._get_kube_df_file(app_info)
app_data = {}
kubernetes_yaml = app_info['app_yaml']
pod_name = self._get_pod_name(app_info)
service_name = pod_name
container_port = self._get_container_port(app_info)
if not container_port:
container_port = 80
df_file = df_file + (
"\n"
"WORKDIR /src \n"
"RUN kubectl create -f {kubernetes_yaml} \ \n"
" && kubectl expose pod {pod_name} --name {service_name} --type LoadBalancer --port 80 --target-port={container_port} --protocol TCP"
).format(kubernetes_yaml=kubernetes_yaml,
pod_name=pod_name,
service_name=service_name,
container_port=container_port)
cont_name = app_info['app_name'] + "-deploy"
app_dir = app_info['app_location']
app_folder_name = app_info['app_folder_name']
df_dir = app_dir + "/" + app_folder_name
df_name = df_dir + "/Dockerfile.deploy"
fp = open(df_name, "w")
fp.write(df_file)
fp.close()
err, output = self.docker_handler.build_container_image(
cont_name, df_name, df_context=df_dir)
if err:
error_output = common_functions.filter_error_output(output)
error_msg = (
"Error encountered in building Dockerfile.deploy {e}").format(
e=err)
error_msg = error_msg + " " + error_output
fmlogger.error(error_msg)
raise exceptions.AppDeploymentFailure(error_msg)
app_details = {}
app_url, status = self._check_if_app_is_ready(app_id, service_name,
app_details)
app_data['status'] = status
app_details['app_url'] = app_url
app_details['pod_name'] = [pod_name]
app_details['service_name'] = service_name
app_details['app_folder_name'] = app_info['app_folder_name']
app_details['env_name'] = app_info['env_name']
app_data['output_config'] = str(app_details)
app_db.App().update(app_id, app_data)
self.docker_handler.remove_container_image(cont_name)
def _check_if_app_is_ready(self, app_id, service_name, app_details):
fmlogger.debug("Checking if application is ready.")
core_v1_api = client.CoreV1Api()
app_ip = ''
while not app_ip:
api_response = core_v1_api.read_namespaced_service(
name=service_name, namespace="default")
if api_response.status.load_balancer.ingress is not None:
app_ip = api_response.status.load_balancer.ingress[0].ip
fmlogger.debug("Service IP:%s" % app_ip)
if app_ip:
break
else:
time.sleep(5)
app_url = "http://" + app_ip
app_details['app_url'] = app_url
app_db.App().update(app_id, {'output_config': str(app_details)})
app_ready = common_functions.is_app_ready(app_url, app_id=app_id)
if app_ready:
return app_url, 'available'
else:
return app_url, 'failed-to-start'
def test_app_update(self):
app_data = {}
app_data['name'] = self._get_app_name()
app_data['location'] = 'abc'
app_data['version'] = 'version'
app_data['dep_target'] = 'local'
app_data['env_id'] = 1
new_app_id = app.App().insert(app_data)
app_data1 = {}
app_data1['version'] = 'version1'
app_data1['location'] = 'location'
app.App().update(new_app_id, app_data1)
updated_app = app.App().get(new_app_id)
self.assertEqual('version1', updated_app.version)
app.App().delete(updated_app.id)
def get_app_type(app_id):
app_obj = app_db.App().get(app_id)
app_yaml_contents = ast.literal_eval(app_obj.app_yaml_contents)
if 'app' in app_yaml_contents:
return 'single-container'
if 'apiVersion' in app_yaml_contents and 'kind' in app_yaml_contents:
return 'multi-container'
return coe_type
def delete_application(self, app_id, app_info):
fmlogger.debug("Deleting application %s" % app_info['app_name'])
app_obj = app_db.App().get(app_id)
try:
app_output_config = ast.literal_eval(app_obj.output_config)
self._delete_service(app_info, app_info['app_name'])
self._delete_deployment(app_info, app_info['app_name'])
# TODO(devdatta) The gcloud sdk is encountering a failure when trying
# to delete the GCR image. So for the time being commenting out below
# call. Send a response to the user asking the user to manually delete
# the image from Google cloud console.
# self._delete_app_image_gcr(tagged_image, app_info)
#tagged_image = app_output_config['tagged_image']
#self._delete_app_image_local(tagged_image)
except Exception as e:
fmlogger.error(e)
app_db.App().delete(app_id)
fmlogger.debug("Done deleting application %s" % app_info['app_name'])
def deploy_application(self, app_id, app_info):
fmlogger.debug("Deploying application %s %s" % (app_id, app_info['app_name']))
env_vars = common_functions.resolve_environment(app_id, app_info)
cont_name = common_functions.get_image_uri(app_info)
app_db.App().update(app_id, {'status': constants.DEPLOYING_APP})
cont_id, app_url = self._deploy_app_container(cont_name, env_vars, app_info)
app_data = {}
app_data['url'] = app_url
app_data['cont_id'] = cont_id.strip()
app_data['app_folder_name'] = app_info['app_folder_name']
app_data['env_name'] = app_info['env_name']
app_db.App().update(app_id, {'output_config': str(app_data)})
app_status = self._check_app_status(app_url)
app_db.App().update(app_id, {'status': app_status,'output_config': str(app_data)})
fmlogger.debug("Done deploying application")
def _validate_host_port(app_info, app_data, env_obj):
app_yaml = common_functions.read_app_yaml(app_info)
# Currently only validating for CloudARK's yaml format
if 'app' not in app_yaml:
return
apps = app_db.App().get_apps_for_env(env_obj.id)
for app in apps:
if app.output_config:
app_config = ast.literal_eval(app.output_config)
if 'host_port' in app_config and 'host_port' in app_yaml['app']:
if app_config['host_port'] == app_yaml['app']['host_port']:
raise exceptions.HostPortConflictException(app_config['host_port'])
def _get_container_names(self, app_name):
pod_name = ''
container_name_set = set()
app_obj = app_db.App().get_by_name(app_name)
app_yaml = ast.literal_eval(app_obj.app_yaml_contents)
if 'kind' in app_yaml and app_yaml['kind'] == 'Pod':
pod_name = app_yaml['metadata']['name']
# There could be multiple documents -- eventually
#for doc in app_yaml:
container_name_set = common_functions.get_cont_names(
app_yaml, container_name_set)
return container_name_set
def get_logs(self, app_id, app_info):
fmlogger.debug("Retrieving logs for application %s %s" %
(app_id, app_info['app_name']))
app_obj = app_db.App().get(app_id)
output_config = ast.literal_eval(app_obj.output_config)
pod_name_list = output_config['pod_name']
log_list = []
for pod in pod_name_list:
logs_path_list = self._retrieve_logs(app_info,
pod,
app_name=app_info['app_name'])
log_list.extend(logs_path_list)
return log_list
def _delete_app(self):
cloud = self.app_info['target']
if cloud == 'aws':
AppHandler.registered_cloud_handlers['aws'].delete_application(
self.app_id, self.app_info)
elif cloud == 'gcloud':
AppHandler.registered_cloud_handlers['gcloud'].delete_application(
self.app_id, self.app_info)
elif cloud == 'local':
AppHandler.registered_cloud_handlers['local'].delete_application(
self.app_id, self.app_info)
else:
fmlogging.error("Unknown deployment target %s" % cloud)
app_db.App().delete(self.app_id)
return
def get_logs(self, app_id, app_info):
fmlogger.debug("Retrieving logs for application %s %s" % (app_id, app_info['app_name']))
app_obj = app_db.App().get(app_id)
output_config = ast.literal_eval(app_obj.output_config)
cont_id = output_config['cont_id'].strip()
logs = self.docker_handler.get_logs(cont_id)
all_lines = []
for log_line in logs:
lines = log_line.split('\n')
all_lines.extend(lines)
return all_lines
def create_service(self, app_name, container_port, host_port, vpc_id,
subnet_list, sec_group_id, cluster_name, task_def_arn,
container_name):
app_url = ''
fmlogger.debug("Creating ECS service for app %s" % app_name)
sec_group_list = [sec_group_id]
lb_arn, lb_dns = self._create_application_lb(app_name, subnet_list,
sec_group_list)
target_group_arn, target_group_name = self._create_target_group(
app_name, container_port, vpc_id)
listener_arn = ''
try:
listener_arn = self._create_lb_listener(host_port, lb_arn,
target_group_arn)
except Exception as e:
fmlogging.error(str(e))
raise e
desired_count = 1 # Number of tasks default to 1
role_obj = self.iam_client.get_role(RoleName='EcsServiceRole')
role_arn = role_obj['Role']['Arn']
try:
self.ecs_client.create_service(
cluster=cluster_name,
serviceName=app_name,
taskDefinition=task_def_arn,
loadBalancers=[{
'targetGroupArn': target_group_arn,
# 'loadBalancerName': app_name,
'containerName': container_name,
'containerPort': int(container_port)
}],
desiredCount=desired_count,
role=role_arn)
except Exception as e:
fmlogger.debug(
"Exception encountered in creating ECS service for app %s" % e)
raise e
fmlogger.debug("ECS service creation for app %s done." % app_name)
service_available = False
service_desc = ''
service_message = ''
count = 1
while not service_available and count < constants.TIMEOUT_COUNT:
try:
service_desc = self.ecs_client.describe_services(
cluster=cluster_name, services=[app_name])
pending_count = service_desc['services'][0]['pendingCount']
running_count = service_desc['services'][0]['runningCount']
service_message = service_desc['services'][0]['events'][0][
'message']
app_data = {'status': service_message}
app_db.App().update_by_name(app_name, app_data)
if pending_count == 0 and running_count == desired_count:
service_available = True
app_url = lb_dns + ":" + str(host_port)
break
except Exception as e:
fmlogger.debug(
"Exception encountered in trying to run describe_services. %s"
% e)
count = count + 1
time.sleep(2)
if count == constants.TIMEOUT_COUNT and not service_available:
raise exceptions.ECSServiceCreateTimeout(app_name)
return app_url, lb_arn, target_group_arn, listener_arn
def test_app_get_non_existing_app(self):
app1 = app.App().get(randint(100, 200))
self.assertIsNone(app1)
def _deploy_service(self, app_id, app_info):
app_data[
'status'] = 'File format not supported. Check https://github.com/cloud-ark/cloudark/issues/200'
app_db.App().update(app_id, app_data)
def _retrieve_deploy_logs(self, app_info, app_name):
app_dir = app_info['app_location']
app_folder_name = app_info['app_folder_name']
df_dir = app_dir + "/" + app_folder_name
logs_path = []
try:
self._setup_kube_config(app_info)
except Exception as e:
fmlogger.error(
"Exception encountered in obtaining kube config %s" % e)
app_db.App().update(app_id, {'status': str(e)})
df = self._get_kube_df_file(app_info)
get_logs = "get-deploytimelogs.sh"
get_logs_wrapper = df_dir + "/" + get_logs
#if not os.path.exists(get_logs_wrapper):
fp = open(get_logs_wrapper, "w")
file_content = (
"#!/bin/bash \n"
"kubectl get pods | grep {app_name} | awk '{{print $1}}' | xargs kubectl describe pods"
).format(app_name=app_name)
fp.write(file_content)
fp.flush()
fp.close()
change_perm_command = ("chmod +x {get_logs_wrapper}").format(
get_logs_wrapper=get_logs_wrapper)
os.system(change_perm_command)
logs_wrapper_cmd = ("\n"
"CMD [\"sh\", \"/src/get-deploytimelogs.sh\"] ")
df = df + logs_wrapper_cmd
log_cont_name = app_info['app_name'] + "-get-deploytimelogs"
df_name = df_dir + "/Dockerfile.get-deploytimelogs"
fp = open(df_name, "w")
fp.write(df)
fp.close()
err, output = self.docker_handler.build_container_image(
log_cont_name, df_name, df_context=df_dir)
if err:
error_msg = ("Error {e}").format(e=err)
fmlogger.error(error_msg)
raise Exception(error_msg)
else:
err1, output1 = self.docker_handler.run_container(log_cont_name)
if not err1:
logs_cont_id = output1.strip()
logs_output = self.docker_handler.get_logs(logs_cont_id)
logs_dir = ("{app_dir}/logs").format(app_dir=app_dir)
logs_dir_command = ("mkdir {logs_dir}").format(
logs_dir=logs_dir)
os.system(logs_dir_command)
deploy_log = app_info['app_name'] + constants.DEPLOY_LOG
deploy_log_path = logs_dir + "/" + deploy_log
fp1 = open(deploy_log_path, "w")
fp1.writelines(logs_output)
fp1.flush()
fp1.close()
self.docker_handler.stop_container(logs_cont_id)
self.docker_handler.remove_container(logs_cont_id)
self.docker_handler.remove_container_image(log_cont_name)
logs_path.append(deploy_log_path)
return logs_path
def deploy_application(self, app_id, app_info):
fmlogger.debug("Deploying application %s" % app_info['app_name'])
self._copy_creds(app_info)
env_vars = common_functions.resolve_environment(app_id, app_info)
app_details = {}
app_data = {}
app_data['status'] = 'setting-up-kubernetes-config'
app_db.App().update(app_id, app_data)
try:
self._setup_kube_config(app_info)
except Exception as e:
fmlogger.error(
"Exception encountered in obtaining kube config %s" % e)
app_db.App().update(app_id, {'status': str(e)})
app_data['status'] = 'creating-deployment-object'
app_db.App().update(app_id, app_data)
tagged_image = common_functions.get_image_uri(app_info)
deployment_obj = self._create_deployment_object(
app_info, tagged_image, env_vars)
app_data['status'] = 'creating-kubernetes-deployment'
app_db.App().update(app_id, app_data)
try:
self._create_deployment(deployment_obj)
except Exception as e:
fmlogger.error(e)
deployment_obj = self._create_deployment_object(app_info,
tagged_image,
env_vars,
alternate_api=True)
try:
self._create_deployment(deployment_obj, alternate_api=True)
except Exception as e:
fmlogger.error(e)
app_data['status'] = 'deployment-error ' + str(e)
app_db.App().update(app_id, app_data)
return
app_data['status'] = 'creating-kubernetes-service'
app_db.App().update(app_id, app_data)
try:
self._create_service(app_info)
except Exception as e:
fmlogger.error(e)
container_port, host_port = self._get_ports(app_info)
app_details['cluster_name'] = self._get_cluster_name(
app_info['env_id'])
app_details['image_name'] = [tagged_image]
app_details['memory'] = common_functions.get_app_memory(app_info)
app_details['app_folder_name'] = app_info['app_folder_name']
app_details['env_name'] = app_info['env_name']
app_details['container_port'] = container_port
app_details['host_port'] = host_port
app_data['output_config'] = str(app_details)
app_data['status'] = 'creating-kubernetes-service'
app_db.App().update(app_id, app_data)
app_data['status'] = 'waiting-for-app-to-get-ready'
app_db.App().update(app_id, app_data)
app_url, status = self._check_if_app_is_ready(app_id,
app_info['app_name'],
app_details)
fmlogger.debug('Application URL:%s' % app_url)
app_data['status'] = status
app_details['app_url'] = app_url
app_data['output_config'] = str(app_details)
app_db.App().update(app_id, app_data)
fmlogger.debug("Done deploying application %s" % app_info['app_name'])
def get_coe_type_for_app(app_id):
app_obj = app_db.App().get(app_id)
coe_type = get_coe_type(app_obj.env_id)
return coe_type
