def connect_jenkins(account=None):
if account is None:
config_document = get_db_config()
account = dict()
account['url'] = config_document['jenkins_url']
account['username'] = config_document['jenkins_user']
account['password'] = config_document['jenkins_pass']
return JenkinsAPI(account['url'], account['username'], account['password'])
def pipeline_create():
"""
Setup a pipeline for an SCM project with a specific branch
This endpoint is used by the DevOps admin
At the end of successful execution, a Jenkins pipeline for the SCM project is created with required build parameters
---
tags:
- pipeline
parameters:
- in: path
name: scm-url
required: true
description: SCM url for this project
type: string
- in: path
name: scm-branch
required: true
description: SCM branch for this project
type: string
- in: path
name: export-policy
required: false
description: export-policy for this project
type: string
responses:
200:
description: Pipeline has been created successfully
"""
#####
# 1. Validate input form parameters
# 2. Get config document for setting up the pipeline details
# 3. Gather storage details for creating PVC for this pipeline
# 4. Create a Kube PVC (Trident creates a PV and an ONTAP volume, maps it to this PVC. We manage only the PVCs)
# 5. Create Jenkins job
# 6. Setup Jenkins purge job for this pipeline
# 7. Record all pipeline details in database
#####
# Validate input web form parameters from the application
_validate_input_form_params(request.form, ['scm-branch', 'scm-url'])
connect, config = _get_config_from_db()
# Gather storage details for creating PVC
scm_project_url = helpers.sanitize_scm_url(request.form['scm-url'])
if scm_project_url is None:
raise GenericException(406, "Invalid SCM URL provided")
pipeline = {
'name':
'-'.join([
'pipeline',
helpers.extract_name_from_git_url(request.form['scm-url']),
request.form['scm-branch']
]),
'export_policy':
request.form.get(
'export-policy',
'default'), # set default export policy if not specified
'scm_url':
scm_project_url
}
# Create PVC. Once we create a Kube PVC, Trident creates an ONTAP volume and a PV for this PVC
kube = KubernetesAPI.get_instance()
vol_size = "10000" # set default vol size to 10Gig, 10000 in MB
# TODO: Change this to default SC from Kube -- list_all_storage_classes and read annotations to find default
storage_class = config.get('storage_class')
if storage_class == '':
storage_class = None # Don't set SC if SC is not passed in Helm, so that Kube can use the default storage class
pvc_response = kube.create_pvc_resource(vol_name=pipeline['name'],
vol_size=vol_size,
storage_class=storage_class)
if not helpers.verify_successful_response(pvc_response):
raise GenericException(500, "Kubernetes PVC creation error")
if pvc_response['phase'] != 'Bound':
raise GenericException(500, "Kubernetes PVC cannot be bound")
# setup params for Jenkins pipeline job
pipeline_job = helpers.set_jenkins_job_params('ci-pipeline')
pipeline_job['volume_claim_name'] = pvc_response['name']
pipeline_job['scm_url'] = request.form['scm-url']
pipeline_job['scm_branch'] = request.form['scm-branch']
pipeline_job['kube_namespace'] = config['kube_namespace']
# TODO: should this volume_name be populated as part of pvc_response? -
# but might want to handle if PVC creation has failed in KubernetesAPI.py
pipeline_job['volume_name'] = kube.get_volume_name_from_pvc(
pvc_response['name']) # Get associated volume with PVC
# TODO: This cannot be None.
# Validate after bootstrapping, PVCs for all services to be part of the config document.
# Remove this after including validation
if config.get('scm_pvc_name') is None:
pipeline_job['scm_volume_claim'] = kube.get_kube_resource_name(
config['scm_volume'], 'pvc')
purge_job = helpers.set_jenkins_job_params(
'trigger-purge') # setup params for Jenkins purge job
purge_job['kube_namespace'] = config['kube_namespace']
# Create Jenkins CI and purge jobs for this pipeline
# If Jenkins connection fails, delete the Kube PVC created from previous step
try:
jenkins = JenkinsAPI(config['jenkins_url'], config['jenkins_user'],
config['jenkins_pass'])
except Exception as exc:
KubernetesAPI.get_instance().delete_pvc(pvc_response['name'])
raise GenericException(500, "Jenkins connection error: %s" % str(exc))
# If job creation fails, delete the Kube PVC created from previous step
try:
jenkins_job_url = jenkins.create_job(job_name=pipeline['name'],
params=pipeline_job,
form_fields=request.form)
jenkins.create_job(job_name='purge_policy_enforcer',
params=purge_job,
form_fields=None)
except Exception as exc:
KubernetesAPI.get_instance().delete_pvc(pvc_response['name'])
traceback.print_exc()
raise GenericException(500,
"Jenkins Job Creation Error: %s" % str(exc))
# Complete gathering pipeline details
pipeline['pvc'] = pvc_response['name']
pipeline['volume'] = pipeline_job['volume_name']
pipeline['jenkins_url'] = jenkins_job_url
# Record new pipeline in database
# TODO: type=pipeline document
try:
new_pipeline_document = Pipeline(**pipeline)
new_pipeline_document.store(connect)
except Exception as exc:
# If DB operation fails, delete the Jenkins pipeline job, purge job and Kube PVC created from previous step
jenkins.delete_job(pipeline['name'])
KubernetesAPI.get_instance().delete_pvc(pvc_response['name'])
raise GenericException(
500,
"Error recording new project in the DB, please contact your administrator",
"Database Exception" + str(exc))
# TODO: Can we do a better in-page rendering instead of navigating to a raw JSON msg?
return jsonify({'project_name': pipeline['name']}), 200
def project_create():
"""
create project
---
tags:
- project
parameters:
- in: path
name: scm-url
required: true
description: git url for this project
type: string
- in: path
name: scm-branch
required: true
description: git branch for this project
type: string
- in: path
name: export-policy
required: false
description: export-policy for this project
type: string
responses:
200:
description: project was created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = ['scm-branch', 'scm-url']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "SCM URL and SCM Branch are required")
scm_project_url = helpers.sanitize_scm_url(request.form['scm-url'])
if scm_project_url is None:
raise GenericException(406, "Invalid SCM URL provided")
project_name = helpers.extract_name_from_git_url(request.form['scm-url'])
project_name += "-" + request.form['scm-branch']
# Kubernetes does not like _
project_name = helpers.replace_kube_invalid_characters(project_name)
# ONTAP does not like -
project_name_no_dashes = helpers.replace_ontap_invalid_char(project_name)
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
vol_uid = "0"
vol_gid = "0"
vol_size = "10000"
if 'export_policy' in request.form:
vol_export_policy = request.form['export-policy']
else:
vol_export_policy = 'default'
try:
status, vol_size = ontap_instance.create_volume(
project_name_no_dashes, vol_size, vol_uid, vol_gid,
vol_export_policy)
except Exception as e:
error_message = "Unable to create backing ontap volume for pipeline"
logging.error(
"Unable to create backing ontap volume for pipeline:\n %s" %
traceback.format_exc())
raise GenericException(500, error_message)
if not helpers.verify_successful_response(status):
error_message = "Unable to create backing ontap volume for pipeline: "
try:
error = status[0]['error_message'].split('(', 1)[0]
except KeyError:
error = ''
error_message = error_message + error
raise GenericException(500, error_message)
# if volume creation successful, autosupport log
# display a warning if this step fails , we don't want to exit out
try:
pass
# helpers.autosupport(project_name_no_dashes, vol_size)
except Exception as e:
logging.warning("WARNING: Unable to generate autosupport log (%s) " %
str(e))
kube_namespace = 'default'
pv_and_pvc_responses = KubernetesAPI().create_pv_and_pvc(
project_name, vol_size, kube_namespace, ontap_data_ip)
for response in pv_and_pvc_responses:
status.append(response)
if not helpers.verify_successful_response(status):
raise GenericException(500, "Kubernetes PV/PVC Error")
try:
jenkins = JenkinsAPI(config_document['jenkins_url'],
config_document['jenkins_user'],
config_document['jenkins_pass'])
except Exception as exc:
raise GenericException(500, "Jenkins connection error: %s" % str(exc))
params = dict()
params['type'] = 'ci-pipeline'
params['volume_name'] = project_name_no_dashes
params['git_volume'] = config_document['git_volume']
params['service_username'] = config_document['service_username']
params['service_password'] = config_document['service_password']
params['broker_url'] = config_document['web_service_url']
params['container_registry'] = config_document['container_registry']
try:
jenkins.create_job(project_name, params, request.form)
except Exception as exc:
raise GenericException(500,
"Jenkins Job Creation Error: %s" % str(exc))
jenkins_url = config_document['jenkins_url'] + "job/" + project_name
# Record new project in database
try:
new_project_document = Project(name=project_name,
volume=project_name_no_dashes,
export_policy=vol_export_policy,
scm_url=scm_project_url,
jenkins_url=jenkins_url)
new_project_document.store(database)
except Exception as exc:
raise GenericException(
500, "Error recording new project in the DB, \
please contact your administrator",
"Database Exception" + str(exc))
# create trigger-purge jenkins job if not already done
jenkins_account = dict()
jenkins_account['url'] = config_document['jenkins_url']
jenkins_account['username'] = config_document['jenkins_user']
jenkins_account['password'] = config_document['jenkins_pass']
try:
helpers.create_purge_jenkins_job(job='purge_policy_enforcer',
account=jenkins_account)
except RuntimeError as exc:
raise GenericException(
500, "Jenkins Job Creation Error: 'purge_policy_enforcer' ")
# need not return project_volume once we start storing volume info in DB
return jsonify({
'project_name': project_name,
'project_volume': project_name_no_dashes
}), 200