def get_services():
'''
Get information about all services associated with Build@Scale
'''
config_document = get_db_config()
kube = KubernetesAPI()
services = []
# Retrieve scm service
scm_service_name = "build-at-scale-%s" % (config_document['scm_type'])
scm_service_url = "http://%s" % kube.get_service_url(scm_service_name)
services.append({
'name': config_document['scm_type'],
'type': 'scm',
'url': scm_service_url
})
# Retrieve registry service
registry_service_name = "build-at-scale-%s" % (
config_document['registry_type'])
registry_service_url = "http://%s" % kube.get_service_url(
registry_service_name)
services.append({
'name': config_document['registry_type'],
'type': 'registry',
'url': registry_service_url
})
# Retrieve ci service
jenkins_service_name = "build-at-scale-jenkins"
jenkins_service_url = "http://%s" % kube.get_service_url(
jenkins_service_name)
services.append({
'name': 'jenkins',
'type': 'ci',
'url': jenkins_service_url
})
# Retrieve database service
database_service_name = "build-at-scale-couchdb"
database_service_url = "http://%s" % kube.get_service_url(
database_service_name)
services.append({
'name': 'couchdb',
'type': 'database',
'url': database_service_url
})
# Retrieve ontap nslm information (this is not a kubernetes service)
services.append({
'name': 'ontap',
'type': 'storage',
'name': 'ontap',
'url': "https://%s" % config_document['ontap_api']
})
logging.error(services)
return services
def get_pipelines_for_dashboard():
'''
Get all pipelines available for displaying in dashboard
'''
database = connect_db()
pipeline_documents = Database.get_documents_by_type(database,
doc_type='project')
pipelines_data = list()
jenkins_obj = connect_jenkins()
for pipeline in pipeline_documents:
last_build_status = jenkins_obj.get_last_build_status(
job_name=pipeline['name'])
scm, jenkins = "dummy-url", "dummy-url"
if 'scm_url' in pipeline:
scm = pipeline['scm_url']
if 'jenkins_url' in pipeline:
kube = KubernetesAPI()
external_url = kube.get_service_url('build-at-scale-jenkins')
jenkins = "http://%s/job/%s" % (external_url, pipeline['name'])
pipelines_data.append({
'pipeline_name': pipeline['name'],
'scm_url': scm,
'jenkins_url': jenkins,
'last_build': last_build_status
})
return pipelines_data
def get_services():
"""
Get information about all services associated with Build@Scale
"""
config_document = get_db_config()
kube = KubernetesAPI()
services = []
# Retrieve scm service
scm_service_name = config_document['scm_service_name']
scm_service_url = kube.get_service_url(scm_service_name)
services.append({
'name': config_document['scm_type'],
'type': 'scm',
'url': scm_service_url
})
# Retrieve registry service
registry_service_name = config_document['registry_service_name']
registry_service_url = kube.get_service_url(registry_service_name)
services.append({
'name': config_document['registry_type'],
'type': 'registry',
'url': registry_service_url
})
# Retrieve ci service
jenkins_service_name = config_document['jenkins_service_name']
jenkins_service_url = kube.get_service_url(jenkins_service_name)
services.append({
'name': 'jenkins',
'type': 'ci',
'url': jenkins_service_url
})
# Retrieve database service
database_service_name = config_document['database_service_name']
database_service_url = kube.get_service_url(database_service_name)
services.append({
'name': 'couchdb',
'type': 'database',
'url': database_service_url
})
logging.error(services)
return services
def _complete_kubernetes_setup_for_workspace(workspace, merge=False):
try:
kube = KubernetesAPI()
kube_pvc_pod_response = kube.create_pvc_and_pod(workspace, merge)
except Exception as exc:
logging.error("Unable to create Kubernetes Workspace PVC/Pod: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create Kubernetes Workspace PVC/Pod")
if not helpers.verify_successful_response(kube_pvc_pod_response):
logging.error("Unable to create Kubernetes Workspace PVC/Pod: %s" %
kube_pvc_pod_response)
raise GenericException(
500, "Unable to create Kubernetes Workspace PVC/Pod")
# workspace['clone_name'] is populated from KubernetesAPI (retrieved from PV-PVC mapping)
workspace['clone_mount'] = "/mnt/" + workspace['clone_name']
# Wait for IDE to be ready before returning
# TODO: Change this to wait and proceed only when service is in Ready state (geta an IP assigned)
try:
time.sleep(60)
workspace['ide'] = kube.get_service_url(workspace['service'])
except:
workspace['ide'] = "NA"
logging.warning("WARNING: Unable to retrieve workspace URL")
# Wait for pod to be ready before executing any commands
# TODO: Add logic to proceed only when pod status is 'Running'
# Set git user.email and user.name , we don't care if the command fails
git_user_cmd = 'git config --global user.name %s' % request.form['username']
git_email_cmd = 'git config --global user.email %s' % workspace[
'user_email']
try:
kube.execute_command_in_pod(workspace['pod'], 'default', git_user_cmd)
kube.execute_command_in_pod(workspace['pod'], 'default', git_email_cmd)
except:
logging.warning(
"WARNING: Unable to configure GIT Username/Email on behalf of user: %s"
% traceback.format_exc())
def _setup_couchdb():
# Configure the couchdb cluster
# Configure the couchdb cluster
headers = {'Content-type': 'application/json'}
db_cluster_config = {
"action": "enable_cluster",
"bind_address": "0.0.0.0",
"username": "******",
"password": "******",
"node_count": "1"
}
# Retrieve customer configuration document from database
database = connect_db()
config_document = get_db_config()
print("GOT DEFAULT CONFIGURATION UPDATED AS:: ", str(config_document))
# Empty SCM URL this is a sign that setup has not been done yet
if config_document['scm_url'] is None:
print("ONE TIME SETUP:: CLUSTER COUCHDB")
try:
r = requests.post("%s/_cluster_setup" % app.config['DATABASE_URL'],
json=db_cluster_config,
headers=headers)
except Exception as exc:
raise GenericException(
500,
"Error configuring the couchdb cluster : %s, please contact your administrator"
% str(exc))
# Populate rest of the configuration details from the ENV variables (set by DevOps Admin)
kube = KubernetesAPI()
# ONTAP
config_document['ontap_svm_name'] = app.config['ONTAP_SVM_NAME']
config_document['ontap_aggr_name'] = app.config['ONTAP_AGGR_NAME']
config_document['ontap_data_ip'] = app.config['ONTAP_DATA_IP']
# SCM
config_document['scm_service_name'] = app.config['SCM_SERVICE_NAME']
config_document['scm_url'] = kube.get_service_url(
service_name=app.config['SCM_SERVICE_NAME'])
config_document['scm_pvc_name'] = app.config['SCM_PVC_NAME']
config_document['scm_volume'] = kube.get_volume_name_from_pvc(
pvc_name=app.config['SCM_PVC_NAME'])
# Jenkins
config_document['jenkins_service_name'] = app.config[
'JENKINS_SERVICE_NAME']
config_document['jenkins_url'] = kube.get_service_url(
service_name=app.config['JENKINS_SERVICE_NAME'])
# Database CouchDB
config_document['database_service_name'] = app.config[
'DATABASE_SERVICE_NAME']
# Artifactory
config_document['registry_service_name'] = app.config[
'REGISTRY_SERVICE_NAME']
config_document['registry_type'] = app.config['REGISTRY_TYPE']
# Webservice
config_document['web_service_name'] = app.config['WEB_SERVICE_NAME']
config_document['web_service_url'] = kube.get_service_url(
service_name=app.config['WEB_SERVICE_NAME'])
config_document.store(database)
print("FINAL CONFIG DOCUMENT:: ", str(config_document))
def workspace_create():
"""
create developer workspace pod
---
tags:
- workspace
parameters:
- in: path
name: workspace-name
required: true
description: Name of the workspace being created
type: string
- in: path
name: build-name
required: true
description: build name (e.g. snapshot) from which clone should be created
type: string
- in: path
name: username
required: false
description: username
type: string
- in: path
name: project-name
required: true
description: the project/pipeline name
type: string
responses:
200:
description: workspace created successfully
"""
# Retrieve customer configuration document from database
try:
database = helpers.connect_db()
config_document = helpers.get_db_config()
except Exception as e:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
if not config_document:
raise GenericException(
500,
"Customer configuration document not found, please contact your administrator",
"Database Exception")
expected_keys = [
'workspace-name', 'build-name', 'username', 'project-name'
]
if not helpers.request_validator(request.form, expected_keys):
raise GenericException(
400,
"workspace-name, build-name, project-name and username are required"
)
username = request.form['username']
try:
user_doc = helpers.get_db_user_document(username)
uid = user_doc['uid']
gid = user_doc['gid']
email = user_doc['email']
except:
raise GenericException(
500, "Error retrieving user information from database",
"Database Exception")
try:
exceeded, workspaces = workspace_obj.exceeded_workspace_count_for_user(
uid, config_document['user_workspace_limit'])
except Exception as exc:
logging.warning(
"WARNING: Unable to check user workspace limit (%s) " %
traceback.format_exc())
if exceeded is True:
raise GenericException(
401, "Please delete one or more workspace(s) from %s and re-try" %
workspaces)
# populate the workspace details
namespace = 'default'
workspace = dict()
workspace['project'] = request.form['project-name']
workspace['snapshot'] = request.form['build-name']
volume_name = request.form['volume-name']
workspace['clone'] = volume_name + \
"_workspace" + helpers.return_random_string(4)
workspace['kb_clone_name'] = helpers.replace_kube_invalid_characters(
workspace['clone'])
workspace['uid'] = uid
workspace['gid'] = gid
workspace['username'] = username
workspace['clone_size_mb'] = "900"
workspace['pod_image'] = config_document['workspace_pod_image']
workspace['clone_mount'] = "/mnt/" + workspace['kb_clone_name']
workspace[
'build_cmd'] = "No build commands have been specified for this project"
workspace['service_type'] = config_document['service_type']
try:
ontap_instance = OntapService(config_document['ontap_api'],
config_document['ontap_apiuser'],
config_document['ontap_apipass'],
config_document['ontap_svm_name'],
config_document['ontap_aggr_name'],
config_document['ontap_data_ip'])
ontap_data_ip = ontap_instance.data_ip
status, vol_size = ontap_instance.create_clone(volume_name,
workspace['uid'],
workspace['gid'],
workspace['clone'],
workspace['snapshot'])
except Exception as exc:
logging.error("Unable to create ontap workspace clone volume: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create ontap workspace clone volume")
if not helpers.verify_successful_response(status):
logging.error("ONTAP Clone Creation Error: %s", repr(status))
return render_template('error.html',
error="Workspace clone creation error"), 400
try:
kube = KubernetesAPI()
except Exception as exc:
logging.error("Unable to connect to Kubernetes: %s" %
traceback.format_exc())
raise GenericException(500, "Unable to connect to Kubernetes")
try:
kube_pv_pvc_pod_response = kube.create_pv_and_pvc_and_pod(
workspace, vol_size, 'default', ontap_data_ip)
except Exception as exc:
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
traceback.format_exc())
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
for response in kube_pv_pvc_pod_response:
status.append(response)
if not helpers.verify_successful_response(status):
logging.error("Unable to create Kubernetes Workspace PV/PVC/Pod: %s" %
response)
raise GenericException(
500, "Unable to create Kubernetes Workspace PV/PVC/Pod")
workspace_pod = workspace['kb_clone_name'] + "-pod"
# Record new workspace in database
try:
new_ws_document = Workspace(name=workspace['clone'],
project=workspace['project'],
username=workspace['username'],
uid=workspace['uid'],
gid=workspace['gid'],
parent_snapshot=workspace['snapshot'],
pod_name=workspace_pod)
new_ws_document.store(database)
except Exception:
raise GenericException(
500, "Error recording new workspace in the DB, \
please contact your administrator",
"Database Exception")
# Wait for pod to be ready before executing any commands
time.sleep(15)
# Set git user.email and user.name , we don't care if the command fails
git_user_cmd = ['git', 'config', '--global', 'user.name', username]
git_email_cmd = ['git', 'config', '--global', 'user.email', email]
try:
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_user_cmd)
response = kube.execute_command_in_pod(workspace_pod, namespace,
git_email_cmd)
except:
logging.warning(
"WARNING: Unable to configure GIT Username/Email on behalf of user: %s"
% traceback.format_exc())
# Wait for IDE to be ready before returning
try:
time.sleep(60)
workspace_ide = kube.get_service_url(workspace['kb_clone_name'] +
"-service")
except:
workspace_ide = "NA"
logging.warning("WARNING: Unable to retrieve workspace URL")
message = "Workspace created successfully!"
return render_template('workspace_details.html',
message=message,
ontap_data_ip=ontap_data_ip,
ontap_volume_name=workspace['clone'],
workspace_ide=workspace_ide), 200