def create_pv_and_pvc(self, vol_name, size, ontap_cluster_data_lif):
''' Create PV and PVC enabled to use the volume 'vol_name' '''
size_to_mb = int(size) / 1024 / 1024
vol_name_no_underscore = helpers.replace_kube_invalid_characters(
vol_name)
pv_status = self.create_pv(vol_name_no_underscore, size_to_mb,
ontap_cluster_data_lif)
if pv_status['code'] == 201:
# wait for PV to be ready!
phase = ""
counter = 0
while phase != 'Available' and counter < 10:
counter += 1
sleep(1)
status = self.read_status("pv", pv_status['resource_name'])
phase = status.status.phase
pv_status['time'] = counter
pvc_status = self.create_pvc(vol_name_no_underscore, size_to_mb)
if pvc_status['code'] == 201:
# wait for PVC to be ready!
phase = ""
counter = 0
while phase != 'Bound' and counter < 10:
counter += 1
sleep(1)
# removed direct call to api.pvc_status
status = self.read_status("pvc", pvc_status['resource_name'])
phase = status.status.phase
pvc_status['time'] = counter
return [pv_status, pvc_status]
def get_kube_resource_name(self, name, resource):
"""
Suffix the resource name with kubernetes resource type
Choices for resource include 'pvc', 'service', 'pod'
:param name: Name of the Kube resource
:param resource: Type of the Kube resource
:return: string representing the resource-name
"""
kube_name = helpers.replace_kube_invalid_characters(name)
return kube_name + '-' + resource
def workspace_create():
"""
create developer workspace pod
---
tags:
- workspace
parameters:
- in: path
name: workspace-name
required: true
description: Name of the workspace being created
type: string
- in: path
name: build-name
required: true
description: build name (e.g. snapshot) from which clone should be created
type: string
- in: path
name: username
required: false
description: username
type: string
- in: path
name: project-name
required: true
description: the project/pipeline name
type: string
responses:
200:
description: workspace created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = [
'workspace-name', 'build-name', 'username', 'project-name'
]
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(
400,
"workspace-name, build-name, project-name and username are required"
)
username = request.form['username']
try:
user_doc = helpers.get_db_user_document(username)
uid = user_doc['uid']
gid = user_doc['gid']
email = user_doc['email']
except:
raise GenericException(
500, "Error retrieving user information from database",
"Database Exception")
try:
exceeded, workspaces = workspace_obj.exceeded_workspace_count_for_user(
uid, config_document['user_workspace_limit'])
except Exception as exc:
logging.warning(
"WARNING: Unable to check user workspace limit (%s) " %
traceback.format_exc())
if exceeded is True:
raise GenericException(
401, "Please delete one or more workspace(s) from %s and re-try" %
workspaces)
# populate the workspace details
namespace = 'default'
workspace = dict()
workspace['project'] = request.form['project-name']
workspace['snapshot'] = request.form['build-name']
volume_name = request.form['volume-name']
workspace['clone'] = volume_name + \
"_workspace" + helpers.return_random_string(4)
workspace['kb_clone_name'] = helpers.replace_kube_invalid_characters(
workspace['clone'])
workspace['uid'] = uid
workspace['gid'] = gid
workspace['username'] = username
workspace['clone_size_mb'] = "900"
workspace['pod_image'] = config_document['workspace_pod_image']
workspace['clone_mount'] = "/mnt/" + workspace['kb_clone_name']
workspace[
'build_cmd'] = "No build commands have been specified for this project"
workspace['service_type'] = config_document['service_type']
try:
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
status, vol_size = ontap_instance.create_clone(volume_name,
workspace['uid'],
workspace['gid'],
workspace['clone'],
workspace['snapshot'])
except Exception as exc:
logging.error("Unable to create ontap workspace clone volume: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create ontap workspace clone volume")
if not helpers.verify_successful_response(status):
logging.error("ONTAP Clone Creation Error: %s", repr(status))
return render_template('error.html',
error="Workspace clone creation error"), 400
try:
kube = KubernetesAPI()
except Exception as exc:
logging.error("Unable to connect to Kubernetes: %s" %
traceback.format_exc())
raise GenericException(500, "Unable to connect to Kubernetes")
try:
kube_pv_pvc_pod_response = kube.create_pv_and_pvc_and_pod(
workspace, vol_size, 'default', ontap_data_ip)
except Exception as exc:
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
for response in kube_pv_pvc_pod_response:
status.append(response)
if not helpers.verify_successful_response(status):
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
response)
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
workspace_pod = workspace['kb_clone_name'] + "-pod"
# Record new workspace in database
try:
new_ws_document = Workspace(name=workspace['clone'],
project=workspace['project'],
username=workspace['username'],
uid=workspace['uid'],
gid=workspace['gid'],
parent_snapshot=workspace['snapshot'],
pod_name=workspace_pod)
new_ws_document.store(database)
except Exception:
raise GenericException(
500, "Error recording new workspace in the DB, \
please contact your administrator",
"Database Exception")
# Wait for pod to be ready before executing any commands
time.sleep(15)
# Set git user.email and user.name , we don't care if the command fails
git_user_cmd = ['git', 'config', '--global', 'user.name', username]
git_email_cmd = ['git', 'config', '--global', 'user.email', email]
try:
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_user_cmd)
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_email_cmd)
except:
logging.warning(
"WARNING: Unable to configure GIT Username/Email on behalf of user: %s"
% traceback.format_exc())
# Wait for IDE to be ready before returning
try:
time.sleep(60)
workspace_ide = kube.get_service_url(workspace['kb_clone_name'] +
"-service")
except:
workspace_ide = "NA"
logging.warning("WARNING: Unable to retrieve workspace URL")
message = "Workspace created successfully!"
return render_template('workspace_details.html',
message=message,
ontap_data_ip=ontap_data_ip,
ontap_volume_name=workspace['clone'],
workspace_ide=workspace_ide), 200
def project_create():
"""
create project
---
tags:
- project
parameters:
- in: path
name: scm-url
required: true
description: git url for this project
type: string
- in: path
name: scm-branch
required: true
description: git branch for this project
type: string
- in: path
name: export-policy
required: false
description: export-policy for this project
type: string
responses:
200:
description: project was created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = ['scm-branch', 'scm-url']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "SCM URL and SCM Branch are required")
scm_project_url = helpers.sanitize_scm_url(request.form['scm-url'])
if scm_project_url is None:
raise GenericException(406, "Invalid SCM URL provided")
project_name = helpers.extract_name_from_git_url(request.form['scm-url'])
project_name += "-" + request.form['scm-branch']
# Kubernetes does not like _
project_name = helpers.replace_kube_invalid_characters(project_name)
# ONTAP does not like -
project_name_no_dashes = helpers.replace_ontap_invalid_char(project_name)
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
vol_uid = "0"
vol_gid = "0"
vol_size = "10000"
if 'export_policy' in request.form:
vol_export_policy = request.form['export-policy']
else:
vol_export_policy = 'default'
try:
status, vol_size = ontap_instance.create_volume(
project_name_no_dashes, vol_size, vol_uid, vol_gid,
vol_export_policy)
except Exception as e:
error_message = "Unable to create backing ontap volume for pipeline"
logging.error(
"Unable to create backing ontap volume for pipeline:\n %s" %
traceback.format_exc())
raise GenericException(500, error_message)
if not helpers.verify_successful_response(status):
error_message = "Unable to create backing ontap volume for pipeline: "
try:
error = status[0]['error_message'].split('(', 1)[0]
except KeyError:
error = ''
error_message = error_message + error
raise GenericException(500, error_message)
# if volume creation successful, autosupport log
# display a warning if this step fails , we don't want to exit out
try:
pass
# helpers.autosupport(project_name_no_dashes, vol_size)
except Exception as e:
logging.warning("WARNING: Unable to generate autosupport log (%s) " %
str(e))
kube_namespace = 'default'
pv_and_pvc_responses = KubernetesAPI().create_pv_and_pvc(
project_name, vol_size, kube_namespace, ontap_data_ip)
for response in pv_and_pvc_responses:
status.append(response)
if not helpers.verify_successful_response(status):
raise GenericException(500, "Kubernetes PV/PVC Error")
try:
jenkins = JenkinsAPI(config_document['jenkins_url'],
config_document['jenkins_user'],
config_document['jenkins_pass'])
except Exception as exc:
raise GenericException(500, "Jenkins connection error: %s" % str(exc))
params = dict()
params['type'] = 'ci-pipeline'
params['volume_name'] = project_name_no_dashes
params['git_volume'] = config_document['git_volume']
params['service_username'] = config_document['service_username']
params['service_password'] = config_document['service_password']
params['broker_url'] = config_document['web_service_url']
params['container_registry'] = config_document['container_registry']
try:
jenkins.create_job(project_name, params, request.form)
except Exception as exc:
raise GenericException(500,
"Jenkins Job Creation Error: %s" % str(exc))
jenkins_url = config_document['jenkins_url'] + "job/" + project_name
# Record new project in database
try:
new_project_document = Project(name=project_name,
volume=project_name_no_dashes,
export_policy=vol_export_policy,
scm_url=scm_project_url,
jenkins_url=jenkins_url)
new_project_document.store(database)
except Exception as exc:
raise GenericException(
500, "Error recording new project in the DB, \
please contact your administrator",
"Database Exception" + str(exc))
# create trigger-purge jenkins job if not already done
jenkins_account = dict()
jenkins_account['url'] = config_document['jenkins_url']
jenkins_account['username'] = config_document['jenkins_user']
jenkins_account['password'] = config_document['jenkins_pass']
try:
helpers.create_purge_jenkins_job(job='purge_policy_enforcer',
account=jenkins_account)
except RuntimeError as exc:
raise GenericException(
500, "Jenkins Job Creation Error: 'purge_policy_enforcer' ")
# need not return project_volume once we start storing volume info in DB
return jsonify({
'project_name': project_name,
'project_volume': project_name_no_dashes
}), 200