def get_services(): ''' Get information about all services associated with Build@Scale ''' config_document = get_db_config() kube = KubernetesAPI() services = [] # Retrieve scm service scm_service_name = "build-at-scale-%s" % (config_document['scm_type']) scm_service_url = "http://%s" % kube.get_service_url(scm_service_name) services.append({ 'name': config_document['scm_type'], 'type': 'scm', 'url': scm_service_url }) # Retrieve registry service registry_service_name = "build-at-scale-%s" % ( config_document['registry_type']) registry_service_url = "http://%s" % kube.get_service_url( registry_service_name) services.append({ 'name': config_document['registry_type'], 'type': 'registry', 'url': registry_service_url }) # Retrieve ci service jenkins_service_name = "build-at-scale-jenkins" jenkins_service_url = "http://%s" % kube.get_service_url( jenkins_service_name) services.append({ 'name': 'jenkins', 'type': 'ci', 'url': jenkins_service_url }) # Retrieve database service database_service_name = "build-at-scale-couchdb" database_service_url = "http://%s" % kube.get_service_url( database_service_name) services.append({ 'name': 'couchdb', 'type': 'database', 'url': database_service_url }) # Retrieve ontap nslm information (this is not a kubernetes service) services.append({ 'name': 'ontap', 'type': 'storage', 'name': 'ontap', 'url': "https://%s" % config_document['ontap_api'] }) logging.error(services) return services
def get_pipelines_for_dashboard(): ''' Get all pipelines available for displaying in dashboard ''' database = connect_db() pipeline_documents = Database.get_documents_by_type(database, doc_type='project') pipelines_data = list() jenkins_obj = connect_jenkins() for pipeline in pipeline_documents: last_build_status = jenkins_obj.get_last_build_status( job_name=pipeline['name']) scm, jenkins = "dummy-url", "dummy-url" if 'scm_url' in pipeline: scm = pipeline['scm_url'] if 'jenkins_url' in pipeline: kube = KubernetesAPI() external_url = kube.get_service_url('build-at-scale-jenkins') jenkins = "http://%s/job/%s" % (external_url, pipeline['name']) pipelines_data.append({ 'pipeline_name': pipeline['name'], 'scm_url': scm, 'jenkins_url': jenkins, 'last_build': last_build_status }) return pipelines_data
def get_services(): """ Get information about all services associated with Build@Scale """ config_document = get_db_config() kube = KubernetesAPI() services = [] # Retrieve scm service scm_service_name = config_document['scm_service_name'] scm_service_url = kube.get_service_url(scm_service_name) services.append({ 'name': config_document['scm_type'], 'type': 'scm', 'url': scm_service_url }) # Retrieve registry service registry_service_name = config_document['registry_service_name'] registry_service_url = kube.get_service_url(registry_service_name) services.append({ 'name': config_document['registry_type'], 'type': 'registry', 'url': registry_service_url }) # Retrieve ci service jenkins_service_name = config_document['jenkins_service_name'] jenkins_service_url = kube.get_service_url(jenkins_service_name) services.append({ 'name': 'jenkins', 'type': 'ci', 'url': jenkins_service_url }) # Retrieve database service database_service_name = config_document['database_service_name'] database_service_url = kube.get_service_url(database_service_name) services.append({ 'name': 'couchdb', 'type': 'database', 'url': database_service_url }) logging.error(services) return services
def _complete_kubernetes_setup_for_workspace(workspace, merge=False): try: kube = KubernetesAPI() kube_pvc_pod_response = kube.create_pvc_and_pod(workspace, merge) except Exception as exc: logging.error("Unable to create Kubernetes Workspace PVC/Pod: %s" % traceback.format_exc()) raise GenericException( 500, "Unable to create Kubernetes Workspace PVC/Pod") if not helpers.verify_successful_response(kube_pvc_pod_response): logging.error("Unable to create Kubernetes Workspace PVC/Pod: %s" % kube_pvc_pod_response) raise GenericException( 500, "Unable to create Kubernetes Workspace PVC/Pod") # workspace['clone_name'] is populated from KubernetesAPI (retrieved from PV-PVC mapping) workspace['clone_mount'] = "/mnt/" + workspace['clone_name'] # Wait for IDE to be ready before returning # TODO: Change this to wait and proceed only when service is in Ready state (geta an IP assigned) try: time.sleep(60) workspace['ide'] = kube.get_service_url(workspace['service']) except: workspace['ide'] = "NA" logging.warning("WARNING: Unable to retrieve workspace URL") # Wait for pod to be ready before executing any commands # TODO: Add logic to proceed only when pod status is 'Running' # Set git user.email and user.name , we don't care if the command fails git_user_cmd = 'git config --global user.name %s' % request.form['username'] git_email_cmd = 'git config --global user.email %s' % workspace[ 'user_email'] try: kube.execute_command_in_pod(workspace['pod'], 'default', git_user_cmd) kube.execute_command_in_pod(workspace['pod'], 'default', git_email_cmd) except: logging.warning( "WARNING: Unable to configure GIT Username/Email on behalf of user: %s" % traceback.format_exc())
def _setup_couchdb(): # Configure the couchdb cluster # Configure the couchdb cluster headers = {'Content-type': 'application/json'} db_cluster_config = { "action": "enable_cluster", "bind_address": "0.0.0.0", "username": "******", "password": "******", "node_count": "1" } # Retrieve customer configuration document from database database = connect_db() config_document = get_db_config() print("GOT DEFAULT CONFIGURATION UPDATED AS:: ", str(config_document)) # Empty SCM URL this is a sign that setup has not been done yet if config_document['scm_url'] is None: print("ONE TIME SETUP:: CLUSTER COUCHDB") try: r = requests.post("%s/_cluster_setup" % app.config['DATABASE_URL'], json=db_cluster_config, headers=headers) except Exception as exc: raise GenericException( 500, "Error configuring the couchdb cluster : %s, please contact your administrator" % str(exc)) # Populate rest of the configuration details from the ENV variables (set by DevOps Admin) kube = KubernetesAPI() # ONTAP config_document['ontap_svm_name'] = app.config['ONTAP_SVM_NAME'] config_document['ontap_aggr_name'] = app.config['ONTAP_AGGR_NAME'] config_document['ontap_data_ip'] = app.config['ONTAP_DATA_IP'] # SCM config_document['scm_service_name'] = app.config['SCM_SERVICE_NAME'] config_document['scm_url'] = kube.get_service_url( service_name=app.config['SCM_SERVICE_NAME']) config_document['scm_pvc_name'] = app.config['SCM_PVC_NAME'] config_document['scm_volume'] = kube.get_volume_name_from_pvc( pvc_name=app.config['SCM_PVC_NAME']) # Jenkins config_document['jenkins_service_name'] = app.config[ 'JENKINS_SERVICE_NAME'] config_document['jenkins_url'] = kube.get_service_url( service_name=app.config['JENKINS_SERVICE_NAME']) # Database CouchDB config_document['database_service_name'] = app.config[ 'DATABASE_SERVICE_NAME'] # Artifactory config_document['registry_service_name'] = app.config[ 'REGISTRY_SERVICE_NAME'] config_document['registry_type'] = app.config['REGISTRY_TYPE'] # Webservice config_document['web_service_name'] = app.config['WEB_SERVICE_NAME'] config_document['web_service_url'] = kube.get_service_url( service_name=app.config['WEB_SERVICE_NAME']) config_document.store(database) print("FINAL CONFIG DOCUMENT:: ", str(config_document))
def workspace_create(): """ create developer workspace pod --- tags: - workspace parameters: - in: path name: workspace-name required: true description: Name of the workspace being created type: string - in: path name: build-name required: true description: build name (e.g. snapshot) from which clone should be created type: string - in: path name: username required: false description: username type: string - in: path name: project-name required: true description: the project/pipeline name type: string responses: 200: description: workspace created successfully """ # Retrieve customer configuration document from database try: database = helpers.connect_db() config_document = helpers.get_db_config() except Exception as e: raise GenericException( 500, "Customer configuration document not found, please contact your administrator", "Database Exception") if not config_document: raise GenericException( 500, "Customer configuration document not found, please contact your administrator", "Database Exception") expected_keys = [ 'workspace-name', 'build-name', 'username', 'project-name' ] if not helpers.request_validator(request.form, expected_keys): raise GenericException( 400, "workspace-name, build-name, project-name and username are required" ) username = request.form['username'] try: user_doc = helpers.get_db_user_document(username) uid = user_doc['uid'] gid = user_doc['gid'] email = user_doc['email'] except: raise GenericException( 500, "Error retrieving user information from database", "Database Exception") try: exceeded, workspaces = workspace_obj.exceeded_workspace_count_for_user( uid, config_document['user_workspace_limit']) except Exception as exc: logging.warning( "WARNING: Unable to check user workspace limit (%s) " % traceback.format_exc()) if exceeded is True: raise GenericException( 401, "Please delete one or more workspace(s) from %s and re-try" % workspaces) # populate the workspace details namespace = 'default' workspace = dict() workspace['project'] = request.form['project-name'] workspace['snapshot'] = request.form['build-name'] volume_name = request.form['volume-name'] workspace['clone'] = volume_name + \ "_workspace" + helpers.return_random_string(4) workspace['kb_clone_name'] = helpers.replace_kube_invalid_characters( workspace['clone']) workspace['uid'] = uid workspace['gid'] = gid workspace['username'] = username workspace['clone_size_mb'] = "900" workspace['pod_image'] = config_document['workspace_pod_image'] workspace['clone_mount'] = "/mnt/" + workspace['kb_clone_name'] workspace[ 'build_cmd'] = "No build commands have been specified for this project" workspace['service_type'] = config_document['service_type'] try: ontap_instance = OntapService(config_document['ontap_api'], config_document['ontap_apiuser'], config_document['ontap_apipass'], config_document['ontap_svm_name'], config_document['ontap_aggr_name'], config_document['ontap_data_ip']) ontap_data_ip = ontap_instance.data_ip status, vol_size = ontap_instance.create_clone(volume_name, workspace['uid'], workspace['gid'], workspace['clone'], workspace['snapshot']) except Exception as exc: logging.error("Unable to create ontap workspace clone volume: %s" % traceback.format_exc()) raise GenericException( 500, "Unable to create ontap workspace clone volume") if not helpers.verify_successful_response(status): logging.error("ONTAP Clone Creation Error: %s", repr(status)) return render_template('error.html', error="Workspace clone creation error"), 400 try: kube = KubernetesAPI() except Exception as exc: logging.error("Unable to connect to Kubernetes: %s" % traceback.format_exc()) raise GenericException(500, "Unable to connect to Kubernetes") try: kube_pv_pvc_pod_response = kube.create_pv_and_pvc_and_pod( workspace, vol_size, 'default', ontap_data_ip) except Exception as exc: logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" % traceback.format_exc()) raise GenericException( 500, "Unable to create Kubernetes Workspace PV/PVC/Pod") for response in kube_pv_pvc_pod_response: status.append(response) if not helpers.verify_successful_response(status): logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" % response) raise GenericException( 500, "Unable to create Kubernetes Workspace PV/PVC/Pod") workspace_pod = workspace['kb_clone_name'] + "-pod" # Record new workspace in database try: new_ws_document = Workspace(name=workspace['clone'], project=workspace['project'], username=workspace['username'], uid=workspace['uid'], gid=workspace['gid'], parent_snapshot=workspace['snapshot'], pod_name=workspace_pod) new_ws_document.store(database) except Exception: raise GenericException( 500, "Error recording new workspace in the DB, \ please contact your administrator", "Database Exception") # Wait for pod to be ready before executing any commands time.sleep(15) # Set git user.email and user.name , we don't care if the command fails git_user_cmd = ['git', 'config', '--global', 'user.name', username] git_email_cmd = ['git', 'config', '--global', 'user.email', email] try: response = kube.execute_command_in_pod(workspace_pod, namespace, git_user_cmd) response = kube.execute_command_in_pod(workspace_pod, namespace, git_email_cmd) except: logging.warning( "WARNING: Unable to configure GIT Username/Email on behalf of user: %s" % traceback.format_exc()) # Wait for IDE to be ready before returning try: time.sleep(60) workspace_ide = kube.get_service_url(workspace['kb_clone_name'] + "-service") except: workspace_ide = "NA" logging.warning("WARNING: Unable to retrieve workspace URL") message = "Workspace created successfully!" return render_template('workspace_details.html', message=message, ontap_data_ip=ontap_data_ip, ontap_volume_name=workspace['clone'], workspace_ide=workspace_ide), 200