def pipeline_create():
"""
Setup a pipeline for an SCM project with a specific branch
This endpoint is used by the DevOps admin
At the end of successful execution, a Jenkins pipeline for the SCM project is created with required build parameters
---
tags:
- pipeline
parameters:
- in: path
name: scm-url
required: true
description: SCM url for this project
type: string
- in: path
name: scm-branch
required: true
description: SCM branch for this project
type: string
- in: path
name: export-policy
required: false
description: export-policy for this project
type: string
responses:
200:
description: Pipeline has been created successfully
"""
#####
# 1. Validate input form parameters
# 2. Get config document for setting up the pipeline details
# 3. Gather storage details for creating PVC for this pipeline
# 4. Create a Kube PVC (Trident creates a PV and an ONTAP volume, maps it to this PVC. We manage only the PVCs)
# 5. Create Jenkins job
# 6. Setup Jenkins purge job for this pipeline
# 7. Record all pipeline details in database
#####
# Validate input web form parameters from the application
_validate_input_form_params(request.form, ['scm-branch', 'scm-url'])
connect, config = _get_config_from_db()
# Gather storage details for creating PVC
scm_project_url = helpers.sanitize_scm_url(request.form['scm-url'])
if scm_project_url is None:
raise GenericException(406, "Invalid SCM URL provided")
pipeline = {
'name':
'-'.join([
'pipeline',
helpers.extract_name_from_git_url(request.form['scm-url']),
request.form['scm-branch']
]),
'export_policy':
request.form.get(
'export-policy',
'default'), # set default export policy if not specified
'scm_url':
scm_project_url
}
# Create PVC. Once we create a Kube PVC, Trident creates an ONTAP volume and a PV for this PVC
kube = KubernetesAPI()
vol_size = "10000" # set default vol size to 10Gig, 10000 in MB
storage_class = config.get(
'storage_class', 'basic') # set default storage class if not specified
pvc_response = kube.create_pvc_resource(vol_name=pipeline['name'],
vol_size=vol_size,
storage_class=storage_class)
if not helpers.verify_successful_response(pvc_response):
raise GenericException(500, "Kubernetes PVC creation error")
# setup params for Jenkins pipeline job
pipeline_job = helpers.set_jenkins_job_params('ci-pipeline')
pipeline_job['volume_claim_name'] = pvc_response['name']
pipeline_job['scm_url'] = request.form['scm-url']
pipeline_job['scm_branch'] = request.form['scm-branch']
# TODO: should this volume_name be populated as part of pvc_response? -
# but might want to handle if PVC creation has failed in KubernetesAPI.py
pipeline_job['volume_name'] = kube.get_volume_name_from_pvc(
pvc_response['name']) # Get associated volume with PVC
# TODO: This cannot be None.
# Validate after bootstrapping, PVCs for all services to be part of the config document.
# Remove this after including validation
if config.get('scm_pvc_name') is None:
pipeline_job['scm_volume_claim'] = kube.get_kube_resource_name(
config['scm_volume'], 'pvc')
purge_job = helpers.set_jenkins_job_params(
'trigger-purge') # setup params for Jenkins purge job
# Create Jenkins CI and purge jobs for this pipeline
try:
jenkins = JenkinsAPI(config['jenkins_url'], config['jenkins_user'],
config['jenkins_pass'])
except Exception as exc:
raise GenericException(500, "Jenkins connection error: %s" % str(exc))
try:
jenkins_job_url = jenkins.create_job(job_name=pipeline['name'],
params=pipeline_job,
form_fields=request.form)
jenkins.create_job(job_name='purge_policy_enforcer',
params=purge_job,
form_fields=None)
except Exception as exc:
raise GenericException(500,
"Jenkins Job Creation Error: %s" % str(exc))
# Complete gathering pipeline details
pipeline['pvc'] = pvc_response['name']
pipeline['volume'] = pipeline_job['volume_name']
pipeline['jenkins_url'] = jenkins_job_url
# Record new pipeline in database
# TODO: type=pipeline document
try:
new_pipeline_document = Pipeline(**pipeline)
new_pipeline_document.store(connect)
except Exception as exc:
raise GenericException(
500,
"Error recording new project in the DB, please contact your administrator",
"Database Exception" + str(exc))
# TODO: Can we do a better in-page rendering instead of navigating to a raw JSON msg?
return jsonify({'project_name': pipeline['name']}), 200
def _setup_couchdb():
# Configure the couchdb cluster
# Configure the couchdb cluster
headers = {'Content-type': 'application/json'}
db_cluster_config = {
"action": "enable_cluster",
"bind_address": "0.0.0.0",
"username": "******",
"password": "******",
"node_count": "1"
}
# Retrieve customer configuration document from database
database = connect_db()
config_document = get_db_config()
print("GOT DEFAULT CONFIGURATION UPDATED AS:: ", str(config_document))
# Empty SCM URL this is a sign that setup has not been done yet
if config_document['scm_url'] is None:
print("ONE TIME SETUP:: CLUSTER COUCHDB")
try:
r = requests.post("%s/_cluster_setup" % app.config['DATABASE_URL'],
json=db_cluster_config,
headers=headers)
except Exception as exc:
raise GenericException(
500,
"Error configuring the couchdb cluster : %s, please contact your administrator"
% str(exc))
# Populate rest of the configuration details from the ENV variables (set by DevOps Admin)
kube = KubernetesAPI()
# ONTAP
config_document['ontap_svm_name'] = app.config['ONTAP_SVM_NAME']
config_document['ontap_aggr_name'] = app.config['ONTAP_AGGR_NAME']
config_document['ontap_data_ip'] = app.config['ONTAP_DATA_IP']
# SCM
config_document['scm_service_name'] = app.config['SCM_SERVICE_NAME']
config_document['scm_url'] = kube.get_service_url(
service_name=app.config['SCM_SERVICE_NAME'])
config_document['scm_pvc_name'] = app.config['SCM_PVC_NAME']
config_document['scm_volume'] = kube.get_volume_name_from_pvc(
pvc_name=app.config['SCM_PVC_NAME'])
# Jenkins
config_document['jenkins_service_name'] = app.config[
'JENKINS_SERVICE_NAME']
config_document['jenkins_url'] = kube.get_service_url(
service_name=app.config['JENKINS_SERVICE_NAME'])
# Database CouchDB
config_document['database_service_name'] = app.config[
'DATABASE_SERVICE_NAME']
# Artifactory
config_document['registry_service_name'] = app.config[
'REGISTRY_SERVICE_NAME']
config_document['registry_type'] = app.config['REGISTRY_TYPE']
# Webservice
config_document['web_service_name'] = app.config['WEB_SERVICE_NAME']
config_document['web_service_url'] = kube.get_service_url(
service_name=app.config['WEB_SERVICE_NAME'])
config_document.store(database)
print("FINAL CONFIG DOCUMENT:: ", str(config_document))