def purge_inconsistent_snapshots(volume):
"""
Snapshot consistency check - ONTAP vs DB
Purge inconsistent snapshot documents from DB
i.e. snapshots in DB that do not exist in ONTAP
@return: count of snapshots deleted from DB
"""
config = helpers.get_db_config()
database = helpers.connect_db()
snapshots_in_db = Database.get_snapshots_by_volume(database, volume=volume)
ontap = OntapService(config['ontap_api'], config['ontap_apiuser'],
config['ontap_apipass'], config['ontap_svm_name'],
config['ontap_aggr_name'], config['ontap_data_ip'])
ontap_snapshot_data = ontap.get_snapshot_list(volume)
if not ontap_snapshot_data:
# purge all snapshots from DB and return:
return purge_snapshots_from_db(snapshots_ontap=[],
snapshots_db=snapshots_in_db)
if not snapshots_in_db:
# return if there are no snapshot documents in db
return 0
ontap_snapshots = [snap['snapshot_name'] for snap in ontap_snapshot_data]
return purge_snapshots_from_db(snapshots_ontap=ontap_snapshots,
snapshots_db=snapshots_in_db)
def purge_old_workspaces():
"""
Purge workspaces older than X days
@return: count of workspaces deleted
"""
database = helpers.connect_db()
config = helpers.get_db_config()
projects_in_db = Database.get_documents_by_type(database, doc_type='project')
if not projects_in_db:
return 0
count = 0
deleted_workspaces = list()
for project in projects_in_db:
workspaces_in_project = Database.get_workspaces_by_project(database,
project=project['name'])
for workspace in workspaces_in_project:
# ontap doesn't provide last_access_timestamp for volumes
# hence, snapdiff latest snapshot with snapshot X days older \
# to find if workspace is active
ontap = OntapService(config['ontap_api'], config['ontap_apiuser'], config['ontap_apipass'], config['ontap_svm_name'], config['ontap_aggr_name'], config['ontap_data_ip'])
deleted, error = ontap.get_snapdiff_and_delete(
volume_name=workspace.value,
count=project['workspace_purge_limit'])
# delete inconsistent or old workspace that exceeded purge limit
if error is not None or deleted is True:
workspace_doc = Database.get_document_by_name(database, workspace.value)
database.delete(workspace_doc)
deleted_workspaces.append(workspace.value)
logging.info("Purge: deleted workspace %s from DB",
workspace.value)
count += 1
return count, deleted_workspaces
def purge_snapshots_by_volume(volume, purge_limit):
"""
Purge snapshots per volume
@return: count of snapshots purged
"""
config = helpers.get_db_config()
ontap = OntapService(config['ontap_api'], config['ontap_apiuser'], config['ontap_apipass'],
config['ontap_svm_name'], config['ontap_aggr_name'], config['ontap_data_ip'])
ontap_snapshot_list = ontap.get_snapshot_list(volume)
if ontap_snapshot_list is None:
return 0
delete_count = len(ontap_snapshot_list) - purge_limit
if delete_count <= 0:
return 0
database = helpers.connect_db()
sorted_by_timestamp = sorted(ontap_snapshot_list, key=lambda snap: snap['timestamp'])
delete_snapshot_list = sorted_by_timestamp[:delete_count]
for snap in delete_snapshot_list:
status = ontap.delete_snapshot(volume, snap['snapshot_name'])
if helpers.verify_successful_response(status):
# delete snapshot document from db
doc = Database.get_document_by_name(database, snap['snapshot_name'])
if not doc: # if snapshot to be deleted is not found in DB
logging.info("Purge: snapshot document not found for %s", snap['snapshot_name'])
else:
database.delete(doc)
logging.info("Purge: snapshot deleted from DB and ONTAP: %s",
snap['snapshot_name'])
return delete_count
def snapshot_list(volume_name):
"""
List all snapshots
---
tags:
- snapshot
parameters:
- in: path
name: volume_name
required: true
description: parent volume name to list snapshots
type: string
responses:
200:
description: snapshot was created successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
ontap = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
snapshots = ontap.get_snapshot_list(volume_name)
return jsonify(snapshots)
def build_clones_list(pipeline_name):
"""
List all 'build' clones belonging to a pipeline
---
tags:
- clone
parameters:
- in: path
name: pipeline_name
required: true
description: pipeline name to list clones from
type: string
responses:
200:
description: clones listed successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
# volume_name = helpers.get_volume_name_for_pipeline(pipeline_name)
build_clones = helpers.get_all_builds_with_status_for_pipeline(
pipeline_name)
return jsonify(build_clones)
def exceeded_workspace_count_for_user(username, limit):
'''Verify if user has exceeded workspace limit'''
database = helpers.connect_db()
workspaces = Database.get_workspaces_by_user(database, user=username)
if len(workspaces) >= limit:
return True, workspaces
return False, workspaces
def purge_snapshots_from_db(snapshots_ontap, snapshots_db):
''' purge snapshots present only in snapshot_db but not in snapshots_ontap'''
database = helpers.connect_db()
count = 0
# snapshots_db is a list of rows:
# where each row has row.key=volume_name and row.value=snapshot_name
for snap in snapshots_db:
if snap.value not in snapshots_ontap:
count += 1
database.delete(snap)
logging.info("Purge: inconsistent snapshot %s deleted from db", snap.value)
return count
def workspace_delete():
"""
delete developer workspace pod
---
tags:
- workspace
parameters:
- in: path
name: workspace-name
required: true
description: Name of the workspace to be deleted
type: string
responses:
200:
description: workspace deleted successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = ['workspace-name']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "workspace-name is required")
try:
helpers.delete_workspace(request.form['workspace-name'])
except Exception as exc:
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
traceback.format_exc())
raise GenericException(
500,
"Unable to delete workspace %s" % request.form['workspace-name'])
response_object = {
'status':
'success',
'message':
"Successfully deleted workspace: %s" % request.form['workspace-name']
}
return jsonify(response_object), 200
def snapshot_purge():
"""
Purge snapshots
---
tags:
- snapshot
parameters:
- in: body
name: snapshot_type
required: false
description: type of snapshots to be purged {"scm" or "ci"}
type: string
responses:
200:
description: "web_service app is tied to a singe customer instance.
Calling this purge API will purge both SCM and CI snapshots from this customer instance"
"""
db_connect = helpers.connect_db()
customer_config = helpers.get_db_config()
if not db_connect or not customer_config:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
if 'type' in request.form:
snapshot_type = request.form['type']
count = snapshot.purge(snapshot_type)
if snapshot_type not in ["scm", "ci"]: # invalid type
raise GenericException(
406,
"Invalid snapshot type parameter: accepted values - 'scm', 'ci'"
)
msg = "Purged %s %s snapshots" % (count, snapshot_type)
else:
count_scm = snapshot.purge("scm")
count_ci = snapshot.purge("ci")
msg = "Purged %s SCM and %s CI snapshots" % (count_scm, count_ci)
response = {
'code': 200,
'resource': 'purge',
'customer_instance': app.config['DATABASE_NAME'],
'message': msg,
'status': 'COMPLETED'
}
return jsonify(response)
def purge_ci_snapshots():
"""
Purge CI snapshots
@return: count of CI snapshots purged
"""
database = helpers.connect_db()
# Get all active projects
projects_in_db = Database.get_documents_by_type(database, doc_type="project")
if not projects_in_db:
return 0
count = 0
# For each project, get all snapshot documents
for project in projects_in_db:
purge_inconsistent_snapshots(volume=project['volume'])
count += purge_snapshots_by_volume(project['volume'], project['ci_purge_limit'])
return count
def user_delete():
"""
delete user
---
tags:
- user
parameters:
- in: path
name: username
required: true
description: username
type: string
responses:
200:
description: user was deleted successfully
"""
# Retrieve customer configuration document from database
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
expected_keys = ['username']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "username is required")
username = request.form['username']
# Check if user exists
current_user = helpers.get_db_user_document(username)
if not current_user:
raise GenericException(
500, "Error deleting user from the DB: user does not exist",
"Database Exception")
# Delete user from database
try:
database.delete(current_user)
except Exception as exc:
raise GenericException(
500, "Error deleting user from the DB, \
please contact your administrator",
"Database Exception")
return jsonify({'message':
"user %s deleted successfully " % username}), 200
def user_delete():
"""
delete user
---
tags:
- user
parameters:
- in: path
name: username
required: true
description: username
type: string
responses:
200:
description: user was deleted successfully
"""
# Retrieve customer configuration document from database
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
_validate_input_form_params(request.form, ['username'])
# Check if user exists
current_user = helpers.get_db_user_document(request.form['username'])
if not current_user:
raise GenericException(
500, "Error deleting user from the DB: user does not exist",
"Database Exception")
# Delete user from database
try:
database.delete(current_user)
except Exception as exc:
raise GenericException(
500, "Error deleting user from the DB, \
please contact your administrator",
"Database Exception")
return jsonify({
'message':
"user %s deleted successfully " % request.form['username']
}), 200
def storage_service_level_modify():
"""
Modify Storage service level of the ONTAP volume
---
tags:
- admin
parameters:
- in: body
name: project_name
required: true
description: Name of the project to modify ssl
type: string
- in: body
name: ssl_name
required: true
description: Name of the ssl to be applied: [performance, extreme, value]
type: string
responses:
200:
description: Storage service level of volume has been modified successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
expected_keys = ['project_name', 'ssl_name']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "project_name and ssl_name are required")
volume_name = helpers.replace_ontap_invalid_char(
request.form['project_name'])
helpers.modify_ssl_for_volume(volume_name, request.form['ssl_name'])
return jsonify({
'message':
"Storage service level has been modified to %s for project %s successfully "
% (request.form['ssl_name'], volume_name)
}), 200
def _get_config_from_db():
# TODO: Should this method belong to database.py? Iff Exceptions can be redirected to web server
"""
Helper method to establish a DB connection and fetch the configuration document
Throws an exception if connection fails
:raises GenericException 500 if DB connection fails
:raises GenericException 500 if config document not found
:return: DB Connection handler and Config document from the DB
"""
# Retrieve customer configuration document from database
try:
connector = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as exc:
raise GenericException(500, GenericException.DB_CONNECTION_ERROR,
"Database Exception")
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
return connector, config_document
def storage_service_level_modify():
"""
Modify Storage service level of the ONTAP volume
---
tags:
- admin
parameters:
- in: body
name: project_name
required: true
description: Name of the project to modify ssl
type: string
- in: body
name: ssl_name
required: true
description: Name of the ssl to be applied [performance, extreme, value]
type: string
responses:
200:
description: Storage service level of volume has been modified successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
_validate_input_form_params(request.form, ['project_name', 'ssl_name'])
volume_name = helpers.replace_ontap_invalid_char(
request.form['project_name'])
helpers.modify_ssl_for_volume(volume_name, request.form['ssl_name'])
return jsonify({
'message':
"Storage service level has been modified to %s for project %s successfully "
% (request.form['ssl_name'], volume_name)
}), 200
def snapshot_delete():
"""
Delete snapshot
---
tags:
- snapshot
parameters:
- in: body
name: snapshot_name
required: true
description: name of the snapshot being created
type: string
- in: body
name: volume_name
required: true
description: name of the volume that needs to be snapshot
type: string
responses:
200:
description: snapshot was deleted successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
ontap = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
status = ontap.delete_snapshot(request.form['volume_name'],
request.form['snapshot_name'])
return jsonify(status)
def user_create():
"""
create user
---
tags:
- user
parameters:
- in: path
name: username
required: true
description: username
type: string
- in: path
name: uid
required: true
description: User id
type: integer
- in: path
name: gid
required: true
description: Group id
type: integer
- in: path
name: email
required: true
description: email
type: string
responses:
200:
description: user was created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
except Exception as e:
raise GenericException(
500,
"Unable to connect to couchdb backend database, please contact your administrator"
)
try:
config_document = helpers.get_db_config()
except Exception as e:
config_document = None
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
expected_keys = ['username', 'uid', 'gid', 'email']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "username,uid,gid, and email are required")
username = request.form['username']
uid = request.form['uid']
gid = request.form['gid']
email = request.form['email']
# Check if user already exists
current_user = helpers.get_db_user_document(username)
if current_user:
raise GenericException(
500, "Error recording new user in the DB: username already exists",
"Database Exception")
# Record new user in database
try:
new_user_document = User(name=username, uid=uid, gid=gid, email=email)
new_user_document.store(database)
except Exception as exc:
raise GenericException(
500, "Error recording new user in the DB, \
please contact your administrator",
"Database Exception")
return jsonify({'username': username}), 200
def user_create():
"""
create user
---
tags:
- user
parameters:
- in: path
name: username
required: true
description: username
type: string
- in: path
name: uid
required: true
description: User id
type: integer
- in: path
name: gid
required: true
description: Group id
type: integer
- in: path
name: email
required: true
description: email
type: string
responses:
200:
description: user was created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
except Exception as e:
raise GenericException(
500,
"Unable to connect to couchdb backend database, please contact your administrator"
)
try:
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(500, GenericException.DB_CONNECTION_ERROR,
"Database Exception")
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
_validate_input_form_params(request.form,
['username', 'uid', 'gid', 'email'])
# Check if user already exists
current_user = helpers.get_db_user_document(request.form['username'])
if current_user:
raise GenericException(
500, "Error recording new user in the DB: username already exists",
"Database Exception")
# Record new user in database
try:
new_user_document = User(name=request.form['username'],
uid=request.form['uid'],
gid=request.form['gid'],
email=request.form['email'])
new_user_document.store(database)
except Exception as exc:
raise GenericException(
500, "Error recording new user in the DB, \
please contact your administrator",
"Database Exception")
return jsonify({'username': request.form['username']}), 200
def volume_claim_clone():
"""
Create Kube PVC clone
This method is in place of snapshotting a source volume.
Volume clones will be used instead of snapshots until Trident supports snapshot creation
---
tags:
- volumeclaim
parameters:
- in: body
name: pvc_clone_name
required: true
description: name of the Kube PVC being created (cloned)
type: string
- in: body
name: pvc_source_name
required: true
description: name of the Kube PVC that is being cloned from
type: string
- in: body
name: build_status
required: false
description: specifies whether this clone is of a successful or failed build
type: string
responses:
200:
description: PVC Clone was created successfully
"""
# TODO: document jenkins_build in docstring
# TODO: do we need volume name?
_validate_input_form_params(request.form, [
'pvc_clone_name', 'pvc_source_name', 'build_status', 'jenkins_build',
'volume_name'
])
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(500, GenericException.DB_CONFIG_DOC_NOT_FOUND,
"Database Exception")
build_status = request.form['build_status'] or 'N/A'
if build_status not in ["passed", "failed", "N/A"]:
raise GenericException(
406,
"Invalid build_status type parameter: accepted values - 'passed', 'failed', 'N/A'"
)
# TODO: this name should be created in KubernetesAPI, but currently will impact create_pvc_and_pod()
kube = KubernetesAPI.get_instance()
pvc_clone_name = kube.get_kube_resource_name(
request.form['pvc_clone_name'], 'pvc')
status = kube.create_pvc_clone_resource(
clone=pvc_clone_name, source=request.form['pvc_source_name'])
# record snapshot in db
db_connect = helpers.connect_db()
if not db_connect:
raise GenericException(500, GenericException.DB_CONNECTION_ERROR,
"Database Exception")
# TODO: Replace Snapshot doc with Clone document
# TODO: Do we need volume or pvc_source_name?
snapshot_doc = Snapshot(
name=request.form['pvc_clone_name'],
pvc_name=pvc_clone_name,
# TODO: Why do we need volume? Also, this is not the clone volume name,
# but the parent pipeline volume name which we use later for only querying.
# Reflect key-name appropriately
parent_pipeline_pvc=request.form['pvc_source_name'],
volume=request.form['volume_name'],
pvc=pvc_clone_name,
jenkins_build=request.form['jenkins_build'],
build_status=build_status)
snapshot_doc.store(db_connect)
return jsonify(status)
def workspace_merge():
"""
Merge developer workspace pod
---
tags:
- workspace
parameters:
- in: path
name: workspace-name
required: true
description: Name of the new merge workspace being created
type: string
- in: path
name: build-name
required: true
description: Build name (e.g. snapshot) from which clone should be created
type: string
- in: path
name: username
required: true
description: Username
type: string
- in: path
name: source-workspace-name
required: true
description: Source workspace
type: integer
responses:
200:
description: merge workspace created successfully
"""
# Validate input web form parameters from the application
_validate_input_form_params(
request.form,
['workspace-name', 'build-name', 'username', 'source-workspace-name'])
workspace = _setup_workspace(request.form, merge=True)
# Run the merge commands in the new workspace. source ws will be mounted at /source_workspace/git
# Destination ws will be mounted at /workspace/git
merge_cmd = '/usr/local/bin/build_at_scale_merge.sh /source_workspace/git /workspace/git'
try:
response = KubernetesAPI.get_instance().execute_command_in_pod(
workspace['pod'], merge_cmd)
except:
logging.error("Unable to successfully complete git merge !" %
traceback.format_exc())
raise GenericException(
500,
"Unable to successfully complete merge!. Please contact your administrator"
)
if response == "0":
message = "Merge workspace created successfully!"
logging.info("Response from workspace POD:: %s" % response)
elif response == "1":
message = "Merge workspace created successfully but merge conflicts were found. " \
"Please check the workspace for conflicts which need to be resolved"
logging.warning("Response from workspace POD:: %s" % response)
else:
# If pod operations fail, delete the workspace PVC and the DB document created from previous steps
KubernetesAPI.get_instance().delete_pvc(workspace['pvc'])
db = helpers.connect_db()
db.delete(workspace['name'])
logging.error("Response from workspace POD:: %s" % response)
raise GenericException(
500,
"Unable to successfully create a merged workspace! , please contact your administrator"
)
return render_template('workspace_details.html',
message=message,
ontap_volume_name=workspace['clone_name'],
workspace_ide=workspace['ide']), 200
def project_create():
"""
create project
---
tags:
- project
parameters:
- in: path
name: scm-url
required: true
description: git url for this project
type: string
- in: path
name: scm-branch
required: true
description: git branch for this project
type: string
- in: path
name: export-policy
required: false
description: export-policy for this project
type: string
responses:
200:
description: project was created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = ['scm-branch', 'scm-url']
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(400, "SCM URL and SCM Branch are required")
scm_project_url = helpers.sanitize_scm_url(request.form['scm-url'])
if scm_project_url is None:
raise GenericException(406, "Invalid SCM URL provided")
project_name = helpers.extract_name_from_git_url(request.form['scm-url'])
project_name += "-" + request.form['scm-branch']
# Kubernetes does not like _
project_name = helpers.replace_kube_invalid_characters(project_name)
# ONTAP does not like -
project_name_no_dashes = helpers.replace_ontap_invalid_char(project_name)
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
vol_uid = "0"
vol_gid = "0"
vol_size = "10000"
if 'export_policy' in request.form:
vol_export_policy = request.form['export-policy']
else:
vol_export_policy = 'default'
try:
status, vol_size = ontap_instance.create_volume(
project_name_no_dashes, vol_size, vol_uid, vol_gid,
vol_export_policy)
except Exception as e:
error_message = "Unable to create backing ontap volume for pipeline"
logging.error(
"Unable to create backing ontap volume for pipeline:\n %s" %
traceback.format_exc())
raise GenericException(500, error_message)
if not helpers.verify_successful_response(status):
error_message = "Unable to create backing ontap volume for pipeline: "
try:
error = status[0]['error_message'].split('(', 1)[0]
except KeyError:
error = ''
error_message = error_message + error
raise GenericException(500, error_message)
# if volume creation successful, autosupport log
# display a warning if this step fails , we don't want to exit out
try:
pass
# helpers.autosupport(project_name_no_dashes, vol_size)
except Exception as e:
logging.warning("WARNING: Unable to generate autosupport log (%s) " %
str(e))
kube_namespace = 'default'
pv_and_pvc_responses = KubernetesAPI().create_pv_and_pvc(
project_name, vol_size, kube_namespace, ontap_data_ip)
for response in pv_and_pvc_responses:
status.append(response)
if not helpers.verify_successful_response(status):
raise GenericException(500, "Kubernetes PV/PVC Error")
try:
jenkins = JenkinsAPI(config_document['jenkins_url'],
config_document['jenkins_user'],
config_document['jenkins_pass'])
except Exception as exc:
raise GenericException(500, "Jenkins connection error: %s" % str(exc))
params = dict()
params['type'] = 'ci-pipeline'
params['volume_name'] = project_name_no_dashes
params['git_volume'] = config_document['git_volume']
params['service_username'] = config_document['service_username']
params['service_password'] = config_document['service_password']
params['broker_url'] = config_document['web_service_url']
params['container_registry'] = config_document['container_registry']
try:
jenkins.create_job(project_name, params, request.form)
except Exception as exc:
raise GenericException(500,
"Jenkins Job Creation Error: %s" % str(exc))
jenkins_url = config_document['jenkins_url'] + "job/" + project_name
# Record new project in database
try:
new_project_document = Project(name=project_name,
volume=project_name_no_dashes,
export_policy=vol_export_policy,
scm_url=scm_project_url,
jenkins_url=jenkins_url)
new_project_document.store(database)
except Exception as exc:
raise GenericException(
500, "Error recording new project in the DB, \
please contact your administrator",
"Database Exception" + str(exc))
# create trigger-purge jenkins job if not already done
jenkins_account = dict()
jenkins_account['url'] = config_document['jenkins_url']
jenkins_account['username'] = config_document['jenkins_user']
jenkins_account['password'] = config_document['jenkins_pass']
try:
helpers.create_purge_jenkins_job(job='purge_policy_enforcer',
account=jenkins_account)
except RuntimeError as exc:
raise GenericException(
500, "Jenkins Job Creation Error: 'purge_policy_enforcer' ")
# need not return project_volume once we start storing volume info in DB
return jsonify({
'project_name': project_name,
'project_volume': project_name_no_dashes
}), 200
def workspace_create():
"""
create developer workspace pod
---
tags:
- workspace
parameters:
- in: path
name: workspace-name
required: true
description: Name of the workspace being created
type: string
- in: path
name: build-name
required: true
description: build name (e.g. snapshot) from which clone should be created
type: string
- in: path
name: username
required: false
description: username
type: string
- in: path
name: project-name
required: true
description: the project/pipeline name
type: string
responses:
200:
description: workspace created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = [
'workspace-name', 'build-name', 'username', 'project-name'
]
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(
400,
"workspace-name, build-name, project-name and username are required"
)
username = request.form['username']
try:
user_doc = helpers.get_db_user_document(username)
uid = user_doc['uid']
gid = user_doc['gid']
email = user_doc['email']
except:
raise GenericException(
500, "Error retrieving user information from database",
"Database Exception")
try:
exceeded, workspaces = workspace_obj.exceeded_workspace_count_for_user(
uid, config_document['user_workspace_limit'])
except Exception as exc:
logging.warning(
"WARNING: Unable to check user workspace limit (%s) " %
traceback.format_exc())
if exceeded is True:
raise GenericException(
401, "Please delete one or more workspace(s) from %s and re-try" %
workspaces)
# populate the workspace details
namespace = 'default'
workspace = dict()
workspace['project'] = request.form['project-name']
workspace['snapshot'] = request.form['build-name']
volume_name = request.form['volume-name']
workspace['clone'] = volume_name + \
"_workspace" + helpers.return_random_string(4)
workspace['kb_clone_name'] = helpers.replace_kube_invalid_characters(
workspace['clone'])
workspace['uid'] = uid
workspace['gid'] = gid
workspace['username'] = username
workspace['clone_size_mb'] = "900"
workspace['pod_image'] = config_document['workspace_pod_image']
workspace['clone_mount'] = "/mnt/" + workspace['kb_clone_name']
workspace[
'build_cmd'] = "No build commands have been specified for this project"
workspace['service_type'] = config_document['service_type']
try:
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
status, vol_size = ontap_instance.create_clone(volume_name,
workspace['uid'],
workspace['gid'],
workspace['clone'],
workspace['snapshot'])
except Exception as exc:
logging.error("Unable to create ontap workspace clone volume: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create ontap workspace clone volume")
if not helpers.verify_successful_response(status):
logging.error("ONTAP Clone Creation Error: %s", repr(status))
return render_template('error.html',
error="Workspace clone creation error"), 400
try:
kube = KubernetesAPI()
except Exception as exc:
logging.error("Unable to connect to Kubernetes: %s" %
traceback.format_exc())
raise GenericException(500, "Unable to connect to Kubernetes")
try:
kube_pv_pvc_pod_response = kube.create_pv_and_pvc_and_pod(
workspace, vol_size, 'default', ontap_data_ip)
except Exception as exc:
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
for response in kube_pv_pvc_pod_response:
status.append(response)
if not helpers.verify_successful_response(status):
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
response)
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
workspace_pod = workspace['kb_clone_name'] + "-pod"
# Record new workspace in database
try:
new_ws_document = Workspace(name=workspace['clone'],
project=workspace['project'],
username=workspace['username'],
uid=workspace['uid'],
gid=workspace['gid'],
parent_snapshot=workspace['snapshot'],
pod_name=workspace_pod)
new_ws_document.store(database)
except Exception:
raise GenericException(
500, "Error recording new workspace in the DB, \
please contact your administrator",
"Database Exception")
# Wait for pod to be ready before executing any commands
time.sleep(15)
# Set git user.email and user.name , we don't care if the command fails
git_user_cmd = ['git', 'config', '--global', 'user.name', username]
git_email_cmd = ['git', 'config', '--global', 'user.email', email]
try:
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_user_cmd)
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_email_cmd)
except:
logging.warning(
"WARNING: Unable to configure GIT Username/Email on behalf of user: %s"
% traceback.format_exc())
# Wait for IDE to be ready before returning
try:
time.sleep(60)
workspace_ide = kube.get_service_url(workspace['kb_clone_name'] +
"-service")
except:
workspace_ide = "NA"
logging.warning("WARNING: Unable to retrieve workspace URL")
message = "Workspace created successfully!"
return render_template('workspace_details.html',
message=message,
ontap_data_ip=ontap_data_ip,
ontap_volume_name=workspace['clone'],
workspace_ide=workspace_ide), 200
def snapshot_create():
"""
Create snapshot from volume
---
tags:
- snapshot
parameters:
- in: body
name: snapshot_name
required: true
description: name of the snapshot being created
type: string
- in: body
name: volume_name
required: true
description: name of the volume that needs to be snapshot
type: string
- in: body
name: build_status
required: false
description: specifies whether this snapshot is of a successful or failed build
type: string
responses:
200:
description: snapshot was created successfully
"""
database = helpers.connect_db()
config_document = helpers.get_db_config()
if not config_document:
raise GenericException(
500,
"Customer config doc not found, please contact your administrator",
"Database Exception")
ontap = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
build_status = request.form['build_status'] or 'N/A'
if build_status not in ["passed", "failed", "N/A"]:
raise GenericException(
406,
"Invalid build_status type parameter: accepted values - 'passed', 'failed', 'N/A'"
)
status = ontap.create_snapshot(request.form['volume_name'],
request.form['snapshot_name'])
# record snapshot in db
db_connect = helpers.connect_db()
if not db_connect:
raise GenericException(
500,
"Database connection failure, please contact your administrator",
"Database Exception")
snapshot_doc = Snapshot(name=request.form['snapshot_name'],
volume=request.form['volume_name'],
jenkins_build=request.form['jenkins_build'],
build_status=build_status)
snapshot_doc.store(db_connect)
return jsonify(status)
