def deploy_application(self, app_id, app_info): fmlogger.debug("Deploying application %s" % app_info['app_name']) self._copy_creds(app_info) app_data = {} app_data['status'] = 'setting-up-kubernetes-config' app_db.App().update(app_id, app_data) try: self._setup_kube_config(app_info) except Exception as e: fmlogger.error( "Exception encountered in obtaining kube config %s" % e) app_db.App().update(app_id, {'status': str(e)}) # Resolve environment common_functions.resolve_environment_multicont(app_id, app_info) app_details = {} app_data = {} app_data['status'] = 'deploying' app_db.App().update(app_id, app_data) try: self._deploy(app_id, app_info) fmlogger.debug("Done deploying application %s" % app_info['app_name']) except exceptions.AppDeploymentFailure as e: fmlogger.error(str(e)) app_data['status'] = 'deployment-failed ' + str( e) + " " + e.get_message() app_db.App().update(app_id, app_data)
def test_app_insert(self): app_data = {} app_data['name'] = self._get_app_name() app_data['location'] = 'abc' app_data['version'] = 'version' app_data['dep_target'] = 'local' app_data['env_id'] = 1 new_app_id = app.App().insert(app_data) self.assertIsNotNone(new_app_id, "App not inserted properly") app.App().delete(new_app_id)
def test_app_delete(self): app_data = {} app_data['name'] = self._get_app_name() app_data['location'] = 'abc' app_data['version'] = 'version' app_data['dep_target'] = 'local' app_data['env_id'] = 1 new_app_id = app.App().insert(app_data) app.App().delete(new_app_id) deleted_app = app.App().get(new_app_id) self.assertIsNone(deleted_app)
def delete_application(self, app_id, app_info): fmlogger.debug("Deleting application %s" % app_info['app_name']) app_obj = app_db.App().get(app_id) try: app_output_config = ast.literal_eval(app_obj.output_config) self._delete_service(app_info, app_output_config['service_name']) pod_name_list = app_output_config['pod_name'] for pod in pod_name_list: self._delete_pod(app_info, pod) except Exception as e: fmlogger.error(e) app_db.App().delete(app_id) fmlogger.debug("Done deleting application %s" % app_info['app_name'])
def is_app_ready(app_url, app_id='', timeout=300): ready = False count = 0 num_of_oks = 10 oks = 0 while count < timeout and not ready: try: response = requests.get(app_url) if response.status_code == 200 or response.status_code == 404: oks = oks + 1 if oks == num_of_oks: ready = True break except Exception as e: fmlogging.error(e) count = count + 1 time.sleep(3) # After every 10 counts check if app still exists if app_id: if count % 10 == 0: app_obj = app_db.App().get(app_id) if not app_obj: count = timeout return ready
def delete_application(self, app_id, app_info): fmlogger.debug("Deleting application %s %s" % (app_id, app_info['app_name'])) try: app_db.App().update(app_id, {'status': constants.DELETING_APP}) app_obj = app_db.App().get(app_id) output_config = ast.literal_eval(app_obj.output_config) cont_id = output_config['cont_id'].strip() err, output = self.docker_handler.stop_container(cont_id) if err: fmlogger.debug("Encountered error in stopping container %s." % cont_id) self.docker_handler.remove_container(cont_id) except Exception as e: fmlogger.error(e) app_db.App().delete(app_id) fmlogger.debug("Done deleting application")
def _deploy_pod(self, app_id, app_info): df_file = self._get_kube_df_file(app_info) app_data = {} kubernetes_yaml = app_info['app_yaml'] pod_name = self._get_pod_name(app_info) service_name = pod_name container_port = self._get_container_port(app_info) if not container_port: container_port = 80 df_file = df_file + ( "\n" "WORKDIR /src \n" "RUN kubectl create -f {kubernetes_yaml} \ \n" " && kubectl expose pod {pod_name} --name {service_name} --type LoadBalancer --port 80 --target-port={container_port} --protocol TCP" ).format(kubernetes_yaml=kubernetes_yaml, pod_name=pod_name, service_name=service_name, container_port=container_port) cont_name = app_info['app_name'] + "-deploy" app_dir = app_info['app_location'] app_folder_name = app_info['app_folder_name'] df_dir = app_dir + "/" + app_folder_name df_name = df_dir + "/Dockerfile.deploy" fp = open(df_name, "w") fp.write(df_file) fp.close() err, output = self.docker_handler.build_container_image( cont_name, df_name, df_context=df_dir) if err: error_output = common_functions.filter_error_output(output) error_msg = ( "Error encountered in building Dockerfile.deploy {e}").format( e=err) error_msg = error_msg + " " + error_output fmlogger.error(error_msg) raise exceptions.AppDeploymentFailure(error_msg) app_details = {} app_url, status = self._check_if_app_is_ready(app_id, service_name, app_details) app_data['status'] = status app_details['app_url'] = app_url app_details['pod_name'] = [pod_name] app_details['service_name'] = service_name app_details['app_folder_name'] = app_info['app_folder_name'] app_details['env_name'] = app_info['env_name'] app_data['output_config'] = str(app_details) app_db.App().update(app_id, app_data) self.docker_handler.remove_container_image(cont_name)
def _check_if_app_is_ready(self, app_id, service_name, app_details): fmlogger.debug("Checking if application is ready.") core_v1_api = client.CoreV1Api() app_ip = '' while not app_ip: api_response = core_v1_api.read_namespaced_service( name=service_name, namespace="default") if api_response.status.load_balancer.ingress is not None: app_ip = api_response.status.load_balancer.ingress[0].ip fmlogger.debug("Service IP:%s" % app_ip) if app_ip: break else: time.sleep(5) app_url = "http://" + app_ip app_details['app_url'] = app_url app_db.App().update(app_id, {'output_config': str(app_details)}) app_ready = common_functions.is_app_ready(app_url, app_id=app_id) if app_ready: return app_url, 'available' else: return app_url, 'failed-to-start'
def test_app_update(self): app_data = {} app_data['name'] = self._get_app_name() app_data['location'] = 'abc' app_data['version'] = 'version' app_data['dep_target'] = 'local' app_data['env_id'] = 1 new_app_id = app.App().insert(app_data) app_data1 = {} app_data1['version'] = 'version1' app_data1['location'] = 'location' app.App().update(new_app_id, app_data1) updated_app = app.App().get(new_app_id) self.assertEqual('version1', updated_app.version) app.App().delete(updated_app.id)
def get_app_type(app_id): app_obj = app_db.App().get(app_id) app_yaml_contents = ast.literal_eval(app_obj.app_yaml_contents) if 'app' in app_yaml_contents: return 'single-container' if 'apiVersion' in app_yaml_contents and 'kind' in app_yaml_contents: return 'multi-container' return coe_type
def delete_application(self, app_id, app_info): fmlogger.debug("Deleting application %s" % app_info['app_name']) app_obj = app_db.App().get(app_id) try: app_output_config = ast.literal_eval(app_obj.output_config) self._delete_service(app_info, app_info['app_name']) self._delete_deployment(app_info, app_info['app_name']) # TODO(devdatta) The gcloud sdk is encountering a failure when trying # to delete the GCR image. So for the time being commenting out below # call. Send a response to the user asking the user to manually delete # the image from Google cloud console. # self._delete_app_image_gcr(tagged_image, app_info) #tagged_image = app_output_config['tagged_image'] #self._delete_app_image_local(tagged_image) except Exception as e: fmlogger.error(e) app_db.App().delete(app_id) fmlogger.debug("Done deleting application %s" % app_info['app_name'])
def deploy_application(self, app_id, app_info): fmlogger.debug("Deploying application %s %s" % (app_id, app_info['app_name'])) env_vars = common_functions.resolve_environment(app_id, app_info) cont_name = common_functions.get_image_uri(app_info) app_db.App().update(app_id, {'status': constants.DEPLOYING_APP}) cont_id, app_url = self._deploy_app_container(cont_name, env_vars, app_info) app_data = {} app_data['url'] = app_url app_data['cont_id'] = cont_id.strip() app_data['app_folder_name'] = app_info['app_folder_name'] app_data['env_name'] = app_info['env_name'] app_db.App().update(app_id, {'output_config': str(app_data)}) app_status = self._check_app_status(app_url) app_db.App().update(app_id, {'status': app_status,'output_config': str(app_data)}) fmlogger.debug("Done deploying application")
def _validate_host_port(app_info, app_data, env_obj): app_yaml = common_functions.read_app_yaml(app_info) # Currently only validating for CloudARK's yaml format if 'app' not in app_yaml: return apps = app_db.App().get_apps_for_env(env_obj.id) for app in apps: if app.output_config: app_config = ast.literal_eval(app.output_config) if 'host_port' in app_config and 'host_port' in app_yaml['app']: if app_config['host_port'] == app_yaml['app']['host_port']: raise exceptions.HostPortConflictException(app_config['host_port'])
def _get_container_names(self, app_name): pod_name = '' container_name_set = set() app_obj = app_db.App().get_by_name(app_name) app_yaml = ast.literal_eval(app_obj.app_yaml_contents) if 'kind' in app_yaml and app_yaml['kind'] == 'Pod': pod_name = app_yaml['metadata']['name'] # There could be multiple documents -- eventually #for doc in app_yaml: container_name_set = common_functions.get_cont_names( app_yaml, container_name_set) return container_name_set
def get_logs(self, app_id, app_info): fmlogger.debug("Retrieving logs for application %s %s" % (app_id, app_info['app_name'])) app_obj = app_db.App().get(app_id) output_config = ast.literal_eval(app_obj.output_config) pod_name_list = output_config['pod_name'] log_list = [] for pod in pod_name_list: logs_path_list = self._retrieve_logs(app_info, pod, app_name=app_info['app_name']) log_list.extend(logs_path_list) return log_list
def _delete_app(self): cloud = self.app_info['target'] if cloud == 'aws': AppHandler.registered_cloud_handlers['aws'].delete_application( self.app_id, self.app_info) elif cloud == 'gcloud': AppHandler.registered_cloud_handlers['gcloud'].delete_application( self.app_id, self.app_info) elif cloud == 'local': AppHandler.registered_cloud_handlers['local'].delete_application( self.app_id, self.app_info) else: fmlogging.error("Unknown deployment target %s" % cloud) app_db.App().delete(self.app_id) return
def get_logs(self, app_id, app_info): fmlogger.debug("Retrieving logs for application %s %s" % (app_id, app_info['app_name'])) app_obj = app_db.App().get(app_id) output_config = ast.literal_eval(app_obj.output_config) cont_id = output_config['cont_id'].strip() logs = self.docker_handler.get_logs(cont_id) all_lines = [] for log_line in logs: lines = log_line.split('\n') all_lines.extend(lines) return all_lines
def create_service(self, app_name, container_port, host_port, vpc_id, subnet_list, sec_group_id, cluster_name, task_def_arn, container_name): app_url = '' fmlogger.debug("Creating ECS service for app %s" % app_name) sec_group_list = [sec_group_id] lb_arn, lb_dns = self._create_application_lb(app_name, subnet_list, sec_group_list) target_group_arn, target_group_name = self._create_target_group( app_name, container_port, vpc_id) listener_arn = '' try: listener_arn = self._create_lb_listener(host_port, lb_arn, target_group_arn) except Exception as e: fmlogging.error(str(e)) raise e desired_count = 1 # Number of tasks default to 1 role_obj = self.iam_client.get_role(RoleName='EcsServiceRole') role_arn = role_obj['Role']['Arn'] try: self.ecs_client.create_service( cluster=cluster_name, serviceName=app_name, taskDefinition=task_def_arn, loadBalancers=[{ 'targetGroupArn': target_group_arn, # 'loadBalancerName': app_name, 'containerName': container_name, 'containerPort': int(container_port) }], desiredCount=desired_count, role=role_arn) except Exception as e: fmlogger.debug( "Exception encountered in creating ECS service for app %s" % e) raise e fmlogger.debug("ECS service creation for app %s done." % app_name) service_available = False service_desc = '' service_message = '' count = 1 while not service_available and count < constants.TIMEOUT_COUNT: try: service_desc = self.ecs_client.describe_services( cluster=cluster_name, services=[app_name]) pending_count = service_desc['services'][0]['pendingCount'] running_count = service_desc['services'][0]['runningCount'] service_message = service_desc['services'][0]['events'][0][ 'message'] app_data = {'status': service_message} app_db.App().update_by_name(app_name, app_data) if pending_count == 0 and running_count == desired_count: service_available = True app_url = lb_dns + ":" + str(host_port) break except Exception as e: fmlogger.debug( "Exception encountered in trying to run describe_services. %s" % e) count = count + 1 time.sleep(2) if count == constants.TIMEOUT_COUNT and not service_available: raise exceptions.ECSServiceCreateTimeout(app_name) return app_url, lb_arn, target_group_arn, listener_arn
def test_app_get_non_existing_app(self): app1 = app.App().get(randint(100, 200)) self.assertIsNone(app1)
def _deploy_service(self, app_id, app_info): app_data[ 'status'] = 'File format not supported. Check https://github.com/cloud-ark/cloudark/issues/200' app_db.App().update(app_id, app_data)
def _retrieve_deploy_logs(self, app_info, app_name): app_dir = app_info['app_location'] app_folder_name = app_info['app_folder_name'] df_dir = app_dir + "/" + app_folder_name logs_path = [] try: self._setup_kube_config(app_info) except Exception as e: fmlogger.error( "Exception encountered in obtaining kube config %s" % e) app_db.App().update(app_id, {'status': str(e)}) df = self._get_kube_df_file(app_info) get_logs = "get-deploytimelogs.sh" get_logs_wrapper = df_dir + "/" + get_logs #if not os.path.exists(get_logs_wrapper): fp = open(get_logs_wrapper, "w") file_content = ( "#!/bin/bash \n" "kubectl get pods | grep {app_name} | awk '{{print $1}}' | xargs kubectl describe pods" ).format(app_name=app_name) fp.write(file_content) fp.flush() fp.close() change_perm_command = ("chmod +x {get_logs_wrapper}").format( get_logs_wrapper=get_logs_wrapper) os.system(change_perm_command) logs_wrapper_cmd = ("\n" "CMD [\"sh\", \"/src/get-deploytimelogs.sh\"] ") df = df + logs_wrapper_cmd log_cont_name = app_info['app_name'] + "-get-deploytimelogs" df_name = df_dir + "/Dockerfile.get-deploytimelogs" fp = open(df_name, "w") fp.write(df) fp.close() err, output = self.docker_handler.build_container_image( log_cont_name, df_name, df_context=df_dir) if err: error_msg = ("Error {e}").format(e=err) fmlogger.error(error_msg) raise Exception(error_msg) else: err1, output1 = self.docker_handler.run_container(log_cont_name) if not err1: logs_cont_id = output1.strip() logs_output = self.docker_handler.get_logs(logs_cont_id) logs_dir = ("{app_dir}/logs").format(app_dir=app_dir) logs_dir_command = ("mkdir {logs_dir}").format( logs_dir=logs_dir) os.system(logs_dir_command) deploy_log = app_info['app_name'] + constants.DEPLOY_LOG deploy_log_path = logs_dir + "/" + deploy_log fp1 = open(deploy_log_path, "w") fp1.writelines(logs_output) fp1.flush() fp1.close() self.docker_handler.stop_container(logs_cont_id) self.docker_handler.remove_container(logs_cont_id) self.docker_handler.remove_container_image(log_cont_name) logs_path.append(deploy_log_path) return logs_path
def deploy_application(self, app_id, app_info): fmlogger.debug("Deploying application %s" % app_info['app_name']) self._copy_creds(app_info) env_vars = common_functions.resolve_environment(app_id, app_info) app_details = {} app_data = {} app_data['status'] = 'setting-up-kubernetes-config' app_db.App().update(app_id, app_data) try: self._setup_kube_config(app_info) except Exception as e: fmlogger.error( "Exception encountered in obtaining kube config %s" % e) app_db.App().update(app_id, {'status': str(e)}) app_data['status'] = 'creating-deployment-object' app_db.App().update(app_id, app_data) tagged_image = common_functions.get_image_uri(app_info) deployment_obj = self._create_deployment_object( app_info, tagged_image, env_vars) app_data['status'] = 'creating-kubernetes-deployment' app_db.App().update(app_id, app_data) try: self._create_deployment(deployment_obj) except Exception as e: fmlogger.error(e) deployment_obj = self._create_deployment_object(app_info, tagged_image, env_vars, alternate_api=True) try: self._create_deployment(deployment_obj, alternate_api=True) except Exception as e: fmlogger.error(e) app_data['status'] = 'deployment-error ' + str(e) app_db.App().update(app_id, app_data) return app_data['status'] = 'creating-kubernetes-service' app_db.App().update(app_id, app_data) try: self._create_service(app_info) except Exception as e: fmlogger.error(e) container_port, host_port = self._get_ports(app_info) app_details['cluster_name'] = self._get_cluster_name( app_info['env_id']) app_details['image_name'] = [tagged_image] app_details['memory'] = common_functions.get_app_memory(app_info) app_details['app_folder_name'] = app_info['app_folder_name'] app_details['env_name'] = app_info['env_name'] app_details['container_port'] = container_port app_details['host_port'] = host_port app_data['output_config'] = str(app_details) app_data['status'] = 'creating-kubernetes-service' app_db.App().update(app_id, app_data) app_data['status'] = 'waiting-for-app-to-get-ready' app_db.App().update(app_id, app_data) app_url, status = self._check_if_app_is_ready(app_id, app_info['app_name'], app_details) fmlogger.debug('Application URL:%s' % app_url) app_data['status'] = status app_details['app_url'] = app_url app_data['output_config'] = str(app_details) app_db.App().update(app_id, app_data) fmlogger.debug("Done deploying application %s" % app_info['app_name'])
def get_coe_type_for_app(app_id): app_obj = app_db.App().get(app_id) coe_type = get_coe_type(app_obj.env_id) return coe_type