def connectKube(self):
try:
self.kube = KubeClient(self.kubernetes_config['url'],
token=self.kubernetes_config['api_key'])
except Exception as e:
print "Error connecting to Kubernetes"
print str(e)
sys.exit(1)
def connectKube(self):
try:
self.kube = KubeClient(self.kubernetes_config['url'], token=self.kubernetes_config['api_key'])
except Exception as e:
print "Error connecting to Kubernetes"
print str(e)
sys.exit(1)
class Kubestack(threading.Thread):
log = logging.getLogger("kubestack.Kubestack")
POD_PREFIX = 'jenkins-slave'
def __init__(self, configfile, start_listeners=True):
threading.Thread.__init__(self, name='Kubestack')
self.configfile = configfile
self.jenkins_object = None
self.kube = None
self.image = None
self._stopped = False
self.watermark_sleep = 10
self.demand_listeners = []
self.destroy_listeners = []
self.existing_pods = 0
self.lock = threading.Lock()
self.start_listeners = start_listeners
self.loadConfig()
# stops thread properly
def stop(self):
self._stopped = True
# disconnect listeners
for listener in self.destroy_listeners:
listener['object'].stop()
listener['object'].join()
if listener['type'] == 'zmq':
listener['object'].zmq_context.destroy()
# main thread worker
def run(self):
while not self._stopped:
try:
for demand_listener in self.demand_listeners:
# it returns a dict with object:total_needed
pods = demand_listener['object'].getDemand()
for key, val in pods.items():
total_existing = self.getExistingPods()
current_demand = val - total_existing
# create all pods that we need
if current_demand > 0:
# create all the pods we need
for _ in range(current_demand):
self.createPod(key)
except Exception as e:
print str(e)
self.log.exception("Exception in main loop")
time.sleep(self.watermark_sleep)
# start the configuration with kubernetes
def connectKube(self):
try:
self.kube = KubeClient(self.kubernetes_config['url'],
token=self.kubernetes_config['api_key'])
except Exception as e:
print "Error connecting to Kubernetes"
print str(e)
sys.exit(1)
# start the connection with jenkins
def connectJenkins(self):
try:
self.jenkins_object = jenkins.Jenkins(
self.jenkins_config['external_url'],
username=self.jenkins_config['user'],
password=self.jenkins_config['pass'])
except Exception as e:
print "Error connecting to Jenkins"
print str(e)
sys.exit(1)
# load configuration details
def loadConfig(self):
try:
config = yaml.load(open(self.configfile))
except Exception:
error_message = "Error: cannot find configuration file %s" % self.configfile
print error_message
sys.exit(1)
# label configuration
self.labels = {}
current_labels = config.get('labels', {})
for label in current_labels:
if not set(('name', 'image')).issubset(label) not in label:
self.log.debug("Label configuration is not properly set")
continue
# add label
current_label = {}
current_label['image'] = label['image']
current_label['cpu'] = label.get('cpu', 0)
current_label['memory'] = label.get('memory', 0)
self.labels[label['name']] = current_label
self.jenkins_config = config.get('jenkins', {})
if not set(('internal_url', 'external_url', 'user', 'pass')).issubset(
self.jenkins_config):
print "Jenkins configuration is not properly set"
sys.exit(1)
self.connectJenkins()
self.kubernetes_config = config.get('kubernetes', {})
if not set(('url', 'api_key')).issubset(self.kubernetes_config):
print "Kuberentes configuration is not properly set"
sys.exit(1)
self.connectKube()
# read demand listeners
if self.start_listeners:
demand_listeners = config.get('demand-listeners', [])
for listener in demand_listeners:
if listener['type'] == 'gearman':
# gearman config
if not set(('host', 'port')).issubset(listener):
print "Gearman configuration is not properly set"
sys.exit(1)
listener['object'] = listeners.GearmanClient(
listener['host'], listener['port'])
listener['object'].connect()
elif listener['type'] == 'jenkins_queue':
# just create the listener object
listener['object'] = listeners.JenkinsQueueClient(self)
# add demand listener
self.demand_listeners.append(listener)
# read destroy listeners
dlisteners = config.get('destroy-listeners', [])
for listener in dlisteners:
if listener['type'] == 'zmq':
if not set(('host', 'port')).issubset(listener):
print "ZMQ configuration is not properly set"
sys.exit(1)
listener['object'] = destroy_listeners.ZMQClient(
self, listener['host'], listener['port'])
listener['object'].start()
# add destroy listener
self.destroy_listeners.append(listener)
# returns a list of pods
def getPods(self):
pods = self.kube.get(url='/pods')
pod_list = self.kube.get_json(pods)
return pod_list
# return the number of pods already in the system, locking access
def getExistingPods(self):
self.lock.acquire()
number = self.existing_pods
self.lock.release()
return number
# returns a list of pod templates
def getPodTemplates(self):
pod_templates = self.kube.get(url='/podtemplates')
pod_template_list = self.kube.get_json(pod_templates)
return pod_template_list
# remove a jenkins node that starts with that id
def deleteJenkinsNode(self, pod_id):
nodes = self.jenkins_object.get_nodes()
for node in nodes:
if pod_id in node['name']:
self.jenkins_object.delete_node(node['name'])
return True
return False
# delete a given pod
def deletePod(self, pod_id):
# first remove from jenkins
try:
self.deleteJenkinsNode(pod_id)
except Exception as e:
self.log.debug("Exception removing from jenkins")
print str(e)
# remove from kube independent of the result of jenkins
# if there was a failure on disconnect, when the node
# is deleted, jenkins will remove that automatically
# after a period of time
try:
self.lock.acquire()
status = self.kube.delete(url='/pods/%s' % pod_id)
self.existing_pods -= 1
if self.existing_pods < 0:
self.existing_pods = 0
self.lock.release()
return status
except Exception as e:
print str(e)
self.log.debug("Exception in deleting pods")
return False
# handles completion of a pod
def podCompleted(self, pod_id):
# swarm concats a sufix at the end, we need to get rid of it
fragments = pod_id.split('-')
fragments.pop()
self.deletePod('-'.join(fragments))
# delete pods starting with a given label
def deletePodsByLabel(self, label):
pod_list = self.getPods()
total_available = 0
for pod_item in pod_list['items']:
current_label = pod_item['metadata']['labels']['name']
if label in current_label:
# delete this pod
self.deletePod(pod_item['metadata']['name'])
# delete a given template
def deletePodTemplate(self, template_id):
status = self.kube.delete(url='/podtemplates/%s' % template_id)
return status
# create a pod for a slave with the given label and image
def createPod(self, label):
# first check if label is available
if not label in self.labels:
self.log.debug("Label %s not available" % label)
return False
current_label = self.labels[label]
# cpu and memory
resources = {}
if current_label['cpu'] or current_label['memory']:
resources['limits'] = {}
if current_label['cpu']:
resources['limits']['cpu'] = current_label['cpu']
if current_label['memory']:
resources['limits']['memory'] = current_label['memory']
try:
self.lock.acquire()
pod_id = self.POD_PREFIX + "-%s" % uuid4()
pod_content = {
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": pod_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"spec": {
"containers": [{
"name":
"jenkins-slave-%s" % label,
"image":
current_label['image'],
"resources":
resources,
"command": [
"sh", "-c",
"/usr/local/bin/jenkins-slave.sh -master %s -username %s -password %s -executors 1 -labels %s"
% (self.jenkins_config['internal_url'],
self.jenkins_config['user'],
self.jenkins_config['pass'], label)
]
}]
}
}
result = self.kube.post(url='/pods', json=pod_content)
self.existing_pods += 1
self.lock.release()
return result
except Exception as e:
self.lock.release()
print str(e)
self.log.exception("Exception creating pod")
return False
# create a template for a slave with the given label and image
def createPodTemplate(self, label):
# first check if label is available
if not label in self.labels:
self.log.debug("Label %s not available" % label)
return False
current_label = self.labels[label]
# cpu and memory
resources = {}
if current_label['cpu'] or current_label['memory']:
resources['limits'] = {}
if current_label['cpu']:
resources['limits']['cpu'] = current_label['cpu']
if current_label['memory']:
resources['limits']['memory'] = current_label['memory']
template_id = "jenkins-slave-%s" % uuid4()
template_content = {
"kind": "PodTemplate",
"apiVersion": "v1",
"metadata": {
"name": template_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"template": {
"metadata": {
"name": template_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"spec": {
"containers": [{
"name":
"jenkins-slave-%s" % label,
"image":
current_label['image'],
"resources":
resources,
"command": [
"sh", "-c",
"/usr/local/bin/jenkins-slave.sh -master %s -username %s -password %s -executors 1 -labels %s"
% (self.jenkins_config['internal_url'],
self.jenkins_config['user'],
self.jenkins_config['pass'], label)
]
}]
}
}
}
result = self.kube.post(url='/podtemplates', json=template_content)
return result
class Kubestack(threading.Thread):
log = logging.getLogger("kubestack.Kubestack")
POD_PREFIX = 'jenkins-slave'
def __init__(self, configfile, start_listeners=True):
threading.Thread.__init__(self, name='Kubestack')
self.configfile = configfile
self.jenkins_object = None
self.kube = None
self.image = None
self._stopped = False
self.watermark_sleep = 10
self.demand_listeners = []
self.destroy_listeners = []
self.existing_pods = 0
self.lock = threading.Lock()
self.start_listeners = start_listeners
self.loadConfig()
# stops thread properly
def stop(self):
self._stopped = True
# disconnect listeners
for listener in self.destroy_listeners:
listener['object'].stop()
listener['object'].join()
if listener['type'] == 'zmq':
listener['object'].zmq_context.destroy()
# main thread worker
def run(self):
while not self._stopped:
try:
for demand_listener in self.demand_listeners:
# it returns a dict with object:total_needed
pods = demand_listener['object'].getDemand()
for key, val in pods.items():
total_existing = self.getExistingPods()
current_demand = val - total_existing
# create all pods that we need
if current_demand > 0:
# create all the pods we need
for _ in range(current_demand):
self.createPod(key)
except Exception as e:
print str(e)
self.log.exception("Exception in main loop")
time.sleep(self.watermark_sleep)
# start the configuration with kubernetes
def connectKube(self):
try:
self.kube = KubeClient(self.kubernetes_config['url'], token=self.kubernetes_config['api_key'])
except Exception as e:
print "Error connecting to Kubernetes"
print str(e)
sys.exit(1)
# start the connection with jenkins
def connectJenkins(self):
try:
self.jenkins_object = jenkins.Jenkins(self.jenkins_config['external_url'],
username=self.jenkins_config['user'], password=self.jenkins_config['pass'])
except Exception as e:
print "Error connecting to Jenkins"
print str(e)
sys.exit(1)
# load configuration details
def loadConfig(self):
try:
config = yaml.load(open(self.configfile))
except Exception:
error_message = "Error: cannot find configuration file %s" % self.configfile
print error_message
sys.exit(1)
# label configuration
self.labels = {}
current_labels = config.get('labels', {})
for label in current_labels:
if not set(('name', 'image')).issubset(label) not in label:
self.log.debug("Label configuration is not properly set")
continue
# add label
current_label = {}
current_label['image'] = label['image']
current_label['cpu'] = label.get('cpu', 0)
current_label['memory'] = label.get('memory', 0)
self.labels[label['name']] = current_label
self.jenkins_config = config.get('jenkins', {})
if not set(('internal_url', 'external_url', 'user', 'pass')).issubset(self.jenkins_config):
print "Jenkins configuration is not properly set"
sys.exit(1)
self.connectJenkins()
self.kubernetes_config = config.get('kubernetes', {})
if not set(('url', 'api_key')).issubset(self.kubernetes_config):
print "Kuberentes configuration is not properly set"
sys.exit(1)
self.connectKube()
# read demand listeners
if self.start_listeners:
demand_listeners = config.get('demand-listeners', [])
for listener in demand_listeners:
if listener['type'] == 'gearman':
# gearman config
if not set(('host', 'port')).issubset(listener):
print "Gearman configuration is not properly set"
sys.exit(1)
listener['object'] = listeners.GearmanClient(listener['host'], listener['port'])
listener['object'].connect()
elif listener['type'] == 'jenkins_queue':
# just create the listener object
listener['object'] = listeners.JenkinsQueueClient(self)
# add demand listener
self.demand_listeners.append(listener)
# read destroy listeners
dlisteners = config.get('destroy-listeners', [])
for listener in dlisteners:
if listener['type'] == 'zmq':
if not set(('host', 'port')).issubset(listener):
print "ZMQ configuration is not properly set"
sys.exit(1)
listener['object'] = destroy_listeners.ZMQClient(self, listener['host'], listener['port'])
listener['object'].start()
# add destroy listener
self.destroy_listeners.append(listener)
# returns a list of pods
def getPods(self):
pods = self.kube.get(url='/pods')
pod_list = self.kube.get_json(pods)
return pod_list
# return the number of pods already in the system, locking access
def getExistingPods(self):
self.lock.acquire()
number = self.existing_pods
self.lock.release()
return number
# returns a list of pod templates
def getPodTemplates(self):
pod_templates = self.kube.get(url='/podtemplates')
pod_template_list = self.kube.get_json(pod_templates)
return pod_template_list
# remove a jenkins node that starts with that id
def deleteJenkinsNode(self, pod_id):
nodes = self.jenkins_object.get_nodes()
for node in nodes:
if pod_id in node['name']:
self.jenkins_object.delete_node(node['name'])
return True
return False
# delete a given pod
def deletePod(self, pod_id):
# first remove from jenkins
try:
self.deleteJenkinsNode(pod_id)
except Exception as e:
self.log.debug("Exception removing from jenkins")
print str(e)
# remove from kube independent of the result of jenkins
# if there was a failure on disconnect, when the node
# is deleted, jenkins will remove that automatically
# after a period of time
try:
self.lock.acquire()
status = self.kube.delete(url='/pods/%s' % pod_id)
self.existing_pods -= 1
if self.existing_pods < 0:
self.existing_pods = 0
self.lock.release()
return status
except Exception as e:
print str(e)
self.log.debug("Exception in deleting pods")
return False
# handles completion of a pod
def podCompleted(self, pod_id):
# swarm concats a sufix at the end, we need to get rid of it
fragments = pod_id.split('-')
fragments.pop()
self.deletePod('-'.join(fragments))
# delete pods starting with a given label
def deletePodsByLabel(self, label):
pod_list = self.getPods()
total_available = 0
for pod_item in pod_list['items']:
current_label = pod_item['metadata']['labels']['name']
if label in current_label:
# delete this pod
self.deletePod(pod_item['metadata']['name'])
# delete a given template
def deletePodTemplate(self, template_id):
status = self.kube.delete(url='/podtemplates/%s' % template_id)
return status
# create a pod for a slave with the given label and image
def createPod(self, label):
# first check if label is available
if not label in self.labels:
self.log.debug("Label %s not available" % label)
return False
current_label = self.labels[label]
# cpu and memory
resources = {}
if current_label['cpu'] or current_label['memory']:
resources['limits'] = {}
if current_label['cpu']:
resources['limits']['cpu'] = current_label['cpu']
if current_label['memory']:
resources['limits']['memory'] = current_label['memory']
try:
self.lock.acquire()
pod_id = self.POD_PREFIX + "-%s" % uuid4()
pod_content = {
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": pod_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"spec": {
"containers": [
{
"name": "jenkins-slave-%s" % label,
"image": current_label['image'],
"resources": resources,
"command": [
"sh",
"-c",
"/usr/local/bin/jenkins-slave.sh -master %s -username %s -password %s -executors 1 -labels %s" %
(self.jenkins_config['internal_url'], self.jenkins_config['user'],
self.jenkins_config['pass'], label)
]
}
]
}
}
result = self.kube.post(url='/pods', json=pod_content)
self.existing_pods += 1
self.lock.release()
return result
except Exception as e:
self.lock.release()
print str(e)
self.log.exception("Exception creating pod")
return False
# create a template for a slave with the given label and image
def createPodTemplate(self, label):
# first check if label is available
if not label in self.labels:
self.log.debug("Label %s not available" % label)
return False
current_label = self.labels[label]
# cpu and memory
resources = {}
if current_label['cpu'] or current_label['memory']:
resources['limits'] = {}
if current_label['cpu']:
resources['limits']['cpu'] = current_label['cpu']
if current_label['memory']:
resources['limits']['memory'] = current_label['memory']
template_id = "jenkins-slave-%s" % uuid4()
template_content = {
"kind": "PodTemplate",
"apiVersion": "v1",
"metadata": {
"name": template_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"template": {
"metadata": {
"name": template_id,
"labels": {
"name": "jenkins-slave-%s" % label,
}
},
"spec": {
"containers": [
{
"name": "jenkins-slave-%s" % label,
"image": current_label['image'],
"resources": resources,
"command": [
"sh",
"-c",
"/usr/local/bin/jenkins-slave.sh -master %s -username %s -password %s -executors 1 -labels %s" %
(self.jenkins_config['internal_url'], self.jenkins_config['user'],
self.jenkins_config['pass'], label)
]
}
]
}
}
}
result = self.kube.post(url='/podtemplates', json=template_content)
return result
