def pipeline_create(): """ Setup a pipeline for an SCM project with a specific branch This endpoint is used by the DevOps admin At the end of successful execution, a Jenkins pipeline for the SCM project is created with required build parameters --- tags: - pipeline parameters: - in: path name: scm-url required: true description: SCM url for this project type: string - in: path name: scm-branch required: true description: SCM branch for this project type: string - in: path name: export-policy required: false description: export-policy for this project type: string responses: 200: description: Pipeline has been created successfully """ ##### # 1. Validate input form parameters # 2. Get config document for setting up the pipeline details # 3. Gather storage details for creating PVC for this pipeline # 4. Create a Kube PVC (Trident creates a PV and an ONTAP volume, maps it to this PVC. We manage only the PVCs) # 5. Create Jenkins job # 6. Setup Jenkins purge job for this pipeline # 7. Record all pipeline details in database ##### # Validate input web form parameters from the application _validate_input_form_params(request.form, ['scm-branch', 'scm-url']) connect, config = _get_config_from_db() # Gather storage details for creating PVC scm_project_url = helpers.sanitize_scm_url(request.form['scm-url']) if scm_project_url is None: raise GenericException(406, "Invalid SCM URL provided") pipeline = { 'name': '-'.join([ 'pipeline', helpers.extract_name_from_git_url(request.form['scm-url']), request.form['scm-branch'] ]), 'export_policy': request.form.get( 'export-policy', 'default'), # set default export policy if not specified 'scm_url': scm_project_url } # Create PVC. Once we create a Kube PVC, Trident creates an ONTAP volume and a PV for this PVC kube = KubernetesAPI() vol_size = "10000" # set default vol size to 10Gig, 10000 in MB storage_class = config.get( 'storage_class', 'basic') # set default storage class if not specified pvc_response = kube.create_pvc_resource(vol_name=pipeline['name'], vol_size=vol_size, storage_class=storage_class) if not helpers.verify_successful_response(pvc_response): raise GenericException(500, "Kubernetes PVC creation error") # setup params for Jenkins pipeline job pipeline_job = helpers.set_jenkins_job_params('ci-pipeline') pipeline_job['volume_claim_name'] = pvc_response['name'] pipeline_job['scm_url'] = request.form['scm-url'] pipeline_job['scm_branch'] = request.form['scm-branch'] # TODO: should this volume_name be populated as part of pvc_response? - # but might want to handle if PVC creation has failed in KubernetesAPI.py pipeline_job['volume_name'] = kube.get_volume_name_from_pvc( pvc_response['name']) # Get associated volume with PVC # TODO: This cannot be None. # Validate after bootstrapping, PVCs for all services to be part of the config document. # Remove this after including validation if config.get('scm_pvc_name') is None: pipeline_job['scm_volume_claim'] = kube.get_kube_resource_name( config['scm_volume'], 'pvc') purge_job = helpers.set_jenkins_job_params( 'trigger-purge') # setup params for Jenkins purge job # Create Jenkins CI and purge jobs for this pipeline try: jenkins = JenkinsAPI(config['jenkins_url'], config['jenkins_user'], config['jenkins_pass']) except Exception as exc: raise GenericException(500, "Jenkins connection error: %s" % str(exc)) try: jenkins_job_url = jenkins.create_job(job_name=pipeline['name'], params=pipeline_job, form_fields=request.form) jenkins.create_job(job_name='purge_policy_enforcer', params=purge_job, form_fields=None) except Exception as exc: raise GenericException(500, "Jenkins Job Creation Error: %s" % str(exc)) # Complete gathering pipeline details pipeline['pvc'] = pvc_response['name'] pipeline['volume'] = pipeline_job['volume_name'] pipeline['jenkins_url'] = jenkins_job_url # Record new pipeline in database # TODO: type=pipeline document try: new_pipeline_document = Pipeline(**pipeline) new_pipeline_document.store(connect) except Exception as exc: raise GenericException( 500, "Error recording new project in the DB, please contact your administrator", "Database Exception" + str(exc)) # TODO: Can we do a better in-page rendering instead of navigating to a raw JSON msg? return jsonify({'project_name': pipeline['name']}), 200
def _setup_couchdb(): # Configure the couchdb cluster # Configure the couchdb cluster headers = {'Content-type': 'application/json'} db_cluster_config = { "action": "enable_cluster", "bind_address": "0.0.0.0", "username": "******", "password": "******", "node_count": "1" } # Retrieve customer configuration document from database database = connect_db() config_document = get_db_config() print("GOT DEFAULT CONFIGURATION UPDATED AS:: ", str(config_document)) # Empty SCM URL this is a sign that setup has not been done yet if config_document['scm_url'] is None: print("ONE TIME SETUP:: CLUSTER COUCHDB") try: r = requests.post("%s/_cluster_setup" % app.config['DATABASE_URL'], json=db_cluster_config, headers=headers) except Exception as exc: raise GenericException( 500, "Error configuring the couchdb cluster : %s, please contact your administrator" % str(exc)) # Populate rest of the configuration details from the ENV variables (set by DevOps Admin) kube = KubernetesAPI() # ONTAP config_document['ontap_svm_name'] = app.config['ONTAP_SVM_NAME'] config_document['ontap_aggr_name'] = app.config['ONTAP_AGGR_NAME'] config_document['ontap_data_ip'] = app.config['ONTAP_DATA_IP'] # SCM config_document['scm_service_name'] = app.config['SCM_SERVICE_NAME'] config_document['scm_url'] = kube.get_service_url( service_name=app.config['SCM_SERVICE_NAME']) config_document['scm_pvc_name'] = app.config['SCM_PVC_NAME'] config_document['scm_volume'] = kube.get_volume_name_from_pvc( pvc_name=app.config['SCM_PVC_NAME']) # Jenkins config_document['jenkins_service_name'] = app.config[ 'JENKINS_SERVICE_NAME'] config_document['jenkins_url'] = kube.get_service_url( service_name=app.config['JENKINS_SERVICE_NAME']) # Database CouchDB config_document['database_service_name'] = app.config[ 'DATABASE_SERVICE_NAME'] # Artifactory config_document['registry_service_name'] = app.config[ 'REGISTRY_SERVICE_NAME'] config_document['registry_type'] = app.config['REGISTRY_TYPE'] # Webservice config_document['web_service_name'] = app.config['WEB_SERVICE_NAME'] config_document['web_service_url'] = kube.get_service_url( service_name=app.config['WEB_SERVICE_NAME']) config_document.store(database) print("FINAL CONFIG DOCUMENT:: ", str(config_document))