def __init__(self, app, nworkers, **kwargs):
# Check if pykube was importable, fail if not
assert KubeConfig is not None, K8S_IMPORT_MESSAGE
runner_param_specs = dict(
k8s_config_path=dict(map=str, default=os_environ.get('KUBECONFIG', None)),
k8s_use_service_account=dict(map=bool, default=False),
k8s_persistent_volume_claim_name=dict(map=str),
k8s_persistent_volume_claim_mount_path=dict(map=str),
k8s_namespace=dict(map=str, default="default"),
k8s_pod_retrials=dict(map=int, valid=lambda x: int > 0, default=3))
if 'runner_param_specs' not in kwargs:
kwargs['runner_param_specs'] = dict()
kwargs['runner_param_specs'].update(runner_param_specs)
"""Start the job runner parent object """
super(KubernetesJobRunner, self).__init__(app, nworkers, **kwargs)
# self.cli_interface = CliInterface()
if "k8s_use_service_account" in self.runner_params and self.runner_params["k8s_use_service_account"]:
self._pykube_api = HTTPClient(KubeConfig.from_service_account())
else:
self._pykube_api = HTTPClient(KubeConfig.from_file(self.runner_params["k8s_config_path"]))
self._galaxy_vol_name = "pvc-galaxy" # TODO this needs to be read from params!!
self._init_monitor_thread()
self._init_worker_threads()
def pykube_client_from_dict(params):
if "k8s_use_service_account" in params and params["k8s_use_service_account"]:
pykube_client = HTTPClient(KubeConfig.from_service_account())
else:
config_path = params.get("k8s_config_path")
if config_path is None:
config_path = os.environ.get('KUBECONFIG', None)
if config_path is None:
config_path = '~/.kube/config'
pykube_client = HTTPClient(KubeConfig.from_file(config_path))
return pykube_client
def _init_kubernetes(self):
self.__logger = logger
self.__logger.debug("Kubernetes auth method: " + self.auth_method)
if self.auth_method == "kubeconfig":
self.__kube_api = HTTPClient(KubeConfig.from_file(self.kubeconfig_path))
elif self.auth_method == "service-account":
self.__kube_api = HTTPClient(KubeConfig.from_service_account())
else:
raise ValueError("Illegal auth_method")
self.job_uuid = str(uuid.uuid4().hex)
now = datetime.utcnow()
self.uu_name = "%s-%s-%s" % (self.name, now.strftime('%Y%m%d%H%M%S'), self.job_uuid[:16])
def deploy_service(template_pod, template_service, branch, domain_zone):
#api = HTTPClient(KubeConfig.from_file("{0}/.kube/config".format(os.environ['HOME'])))
api = HTTPClient(KubeConfig.from_service_account())
with open(template_service) as t_file:
ts = yaml.load(t_file)
svc_name = ts['metadata']['name']
name = "{0}-{1}".format(svc_name, branch)
ts['spec']['type'] = 'LoadBalancer'
ts['spec']['selector']['name'] = name
ts['metadata']['name'] = name
ts['metadata']['labels']['name'] = name
new = Service(api, ts)
new.create()
print "New service created"
with open(template_pod) as t_file:
tp = yaml.load(t_file)
name = tp['metadata']['name']
name = "{0}-{1}".format(name, branch)
image = tp['spec']['containers'][0]['image']
image = "{0}:{1}".format(image, branch)
tp['spec']['containers'][0]['image'] = image
tp['spec']['containers'][0]['name'] = name
tp['metadata']['name'] = name
tp['metadata']['labels']['name'] = name
new = Pod(api, tp)
new.create()
print "New pod created"
print "Waiting for ELB to spawn"
lb_name = get_service_lb.wait_for_lb_name(name)
print "Got ELB {0}".format(lb_name)
return lb_name, svc_name
def load_and_check_config(self):
if not os.path.exists(self.config_path):
print("Config does not exist at path " + self.config_path + "!")
return False
try:
self.config = KubeConfig.from_file(self.config_path)
except:
print("Config at path " + self.config_path + " failed to validate!")
return False
# Check current context
if self.context_override != None:
if self.context_override not in self.config.contexts:
print("Context override " + self.context_override + " not in list of contexts.")
return False
self.config.set_current_context(self.context_override)
elif self.config.current_context == None:
print("Context not set, not sure which to use.")
return False
curr_ctx = self.config.contexts[self.config.current_context]
self.api = HTTPClient(self.config)
if not self.enable_secure:
print('[note] we are in insecure mode, disabling warnings')
requests.packages.urllib3.disable_warnings()
self.api.session.verify = False
return True
def action(self, resource, action_name):
api = HTTPClient(KubeConfig.from_file("~/.kube/config"))
log.debug('Executing %s %s',
action_name, resource.name)
# XXX: self._configs is used in _compile_action_file via _make_args. It has to be here
self._configs = self.prepare_configs(resource)
action_file = self._compile_action_file(resource, action_name)
log.debug('action_file: %s', action_file)
# XXX: seems hacky
obj = yaml.load(open(action_file).read())
k8s_class = obj['kind']
if action_name == 'run':
k8s_class = getattr(pykube.objects, k8s_class)
k8s_obj = k8s_class(api, obj)
k8s_obj.create()
self._wait_for(k8s_obj)
elif action_name == 'update':
k8s_class = getattr(pykube.objects, k8s_class)
k8s_obj = k8s_class(api, obj)
k8s_obj.reload()
# generate new data
new_data = self._compile_action_file(resource, 'run')
new_obj = yaml.load(open(new_data).read())
_update_obj(k8s_obj.obj, new_obj)
# hacky
pykube.objects.jsonpatch.make_patch = jsondiff.make
k8s_obj.update()
self._wait_for(k8s_obj)
elif action_name == 'delete':
raise NotImplemented(action_name)
else:
raise NotImplemented(action_name)
def loadconfig(self):
if not self.api is None:
return
logger.debug("Loading kubeconfig...")
try:
self.kubeconfig = KubeConfig.from_file(env("LOCAL_KUBECONFIG_PATH", "/azk/deploy/.kube/config"))
self.api = HTTPClient(self.kubeconfig)
self.namespace = env("KUBE_NAMESPACE")
self.context = env("KUBE_CONTEXT")
if self.context is None:
if "current-context" in self.kubeconfig.doc:
self.context = self.kubeconfig.doc["current-context"]
else:
logger.fatal("KUBE_CONTEXT in env is not set and current-context is not set in kubeconfig.")
exit(1)
if self.context not in self.kubeconfig.contexts:
logger.fatal("Context '" + str(self.context) + "' is not found in kubeconfig.")
exit(1)
self.kubeconfig.set_current_context(self.context)
logger.debug("Testing connectivity...")
if self.namespace is None and "namespace" in self.kubeconfig.contexts[self.context]:
self.namespace = self.kubeconfig.contexts[self.context]["namespace"]
if self.namespace is None:
logger.fatal("KUBE_NAMESPACE is not set and there is no namespace set in kubeconfig context " + str(self.kubeconfig.current_context) + ".")
exit(1)
pods = Pod.objects(self.api).filter(namespace=self.namespace)
logger.info("Currently " + str(len(pods)) + " pods in '" + self.namespace + "' namespace, kubernetes connection appears to be working.")
except Exception as e:
logger.fatal("Unable to load kubeconfig/connection failed, " + str(e.strerror))
exit(1)
def test_fail_job(self):
fail = FailJob()
self.assertRaises(RuntimeError, fail.run)
# Check for retrials
kube_api = HTTPClient(KubeConfig.from_file("~/.kube/config")) # assumes minikube
jobs = Job.objects(kube_api).filter(selector="luigi_task_id="
+ fail.job_uuid)
self.assertEqual(len(jobs.response["items"]), 1)
job = Job(kube_api, jobs.response["items"][0])
self.assertTrue("failed" in job.obj["status"])
self.assertTrue(job.obj["status"]["failed"] > fail.max_retrials)
def _get_api_pod(self):
"""Get the pod resource from the API.
:return: Dictionary representation of Pod from k8s API.
"""
# If kubeconfig was specified, use the pykube library.
if self.kubeconfig_path:
_log.info("Using kubeconfig at %s", self.kubeconfig_path)
try:
api = HTTPClient(KubeConfig.from_file(self.kubeconfig_path))
pod = Query(api, Pod, self.namespace).get_by_name(self.pod_name)
_log.debug("Found pod: %s: ", pod.obj)
except Exception as e:
raise PolicyException("Error querying Kubernetes API",
details=str(e.message))
else:
return pod.obj
# Otherwise, use direct HTTP query to get pod.
with requests.Session() as session:
if self.auth_token:
_log.debug('Updating header with Token %s', self.auth_token)
session.headers.update({'Authorization':
'Bearer ' + self.auth_token})
# Generate the API endpoint to query.
path = "namespaces/%s/pods/%s" % (self.namespace, self.pod_name)
path = os.path.join(self.api_root, path)
# Perform the API query and handle the result.
try:
_log.debug('Querying Kubernetes API for Pod: %s', path)
if self.client_certificate and self.client_key:
_log.debug("Using client certificate for Query API. "
"cert: %s, key: %s",
self.client_certificate,
self.client_key)
cert = (self.client_certificate,
self.client_key)
response = session.get(path, cert=cert,
verify=self.certificate_authority)
else:
_log.debug('Using direct connection for query API')
response = session.get(path,
verify=self.certificate_authority)
except BaseException, e:
_log.exception("Exception hitting Kubernetes API")
raise ApplyProfileError("Error querying Kubernetes API",
details=str(e.message))
else:
def run():
k8s = HTTPClient(KubeConfig.from_service_account())
mongo_connection_string = os.environ.get('MONGO_CONNECTION_STRING', 'mongodb://127.0.0.1')
logging.info('Mongo server %s', mongo_connection_string)
replica_manager = ReplicaManager(
app_name=os.environ['APP_NAME'],
creator_name=os.environ['CREATOR_NAME'],
hostname=os.environ['MONGO_HOSTNAME'],
k8s=k8s,
local_mongo_server_conn = mongo_connection_string,
external_ip=os.environ['EXTERNAL_IP']
)
replica_manager.start()
def run_all(self):
api = HTTPClient(KubeConfig.from_file('~/.kube/config'))
datas = []
for i, (resource, path, to) in enumerate(self.paths):
datas.append(self.make_confimap_data(resource, path, i))
self.data_sufix = random_string(52)
self.configmap_name = 'configmap' + self.data_sufix
self.configmap_namespace = 'default'
self.configmap_datas = datas
obj = self.make_configmap_obj(datas)
self.configmap_obj = pykube.objects.ConfigMap(api, obj)
self.configmap_obj.create()
log.debug("Created ConfigMap: %s", self.configmap_obj.name)
return
def _get_kubernetes_pod_cidr(self):
"""
Attempt to get the Kubernetes pod CIDR for this node.
First check if we've written it to disk. If so, use that value. If
not, then query the Kubernetes API for it.
"""
_log.info("Getting node.spec.podCidr from API, kubeconfig: %s",
self.kubeconfig_path)
if not self.kubeconfig_path:
# For now, kubeconfig is the only supported auth method.
print_cni_error(ERR_CODE_GENERIC, "Missing kubeconfig",
"usePodCidr requires specification of kubeconfig file")
sys.exit(ERR_CODE_GENERIC)
# Query the API for this node. Default node name to the hostname.
try:
api = HTTPClient(KubeConfig.from_file(self.kubeconfig_path))
node = None
for n in Node.objects(api):
_log.debug("Checking node: %s", n.obj["metadata"]["name"])
if n.obj["metadata"]["name"] == self.k8s_node_name:
node = n
break
if not node:
raise KeyError("Unable to find node in API: %s", self.k8s_node_name)
_log.debug("Found node %s: %s: ", node.obj["metadata"]["name"],
node.obj["spec"])
except Exception:
print_cni_error(ERR_CODE_GENERIC, "Error querying Kubernetes API",
"Failed to get podCidr from Kubernetes API")
sys.exit(ERR_CODE_GENERIC)
else:
pod_cidr = node.obj["spec"].get("podCIDR")
if not pod_cidr:
print_cni_error(ERR_CODE_GENERIC, "Missing podCidr",
"No podCidr for node %s" % self.k8s_node_name)
sys.exit(ERR_CODE_GENERIC)
_log.debug("Using podCidr: %s", pod_cidr)
return pod_cidr
def test():
num = 3
base = 27020
k8s = HTTPClient(KubeConfig.from_service_account())
k8s.url = 'http://127.0.0.1:8001'
k8s.session = k8s.build_session()
def get_mongo_pods():
return [
Pod(None, {
'metadata': {
'labels': {
'hostname': 'fb-1.db.waverbase.com:%d' % p
}
},
'status': {
'podIP': '127.0.0.1:%d' % p
}
}
)
for p in range(base, base+num)
]
for p in range(base, base+num):
replica_manager = ReplicaManager(
app_name='testapp',
creator_name='testcreator',
hostname='fb-1.db.waverbase.com:%d' % p,
k8s=k8s,
local_mongo_server_conn = 'mongodb://127.0.0.1:%d' % p,
external_ip='127.0.0.1:%d' % p
)
replica_manager.local_pod_ip = '127.0.0.1:%d' % p
replica_manager.get_mongo_pods = get_mongo_pods
replica_manager.start()
def setUp(self):
self.cfg = KubeConfig.from_file(GOOD_CONFIG_FILE_PATH)
if str in ['true', 'True']:
return True
return False
def parse_service(service):
data = Bunch(service.annotations)
data.ip = service.obj['spec']['clusterIP']
data.proxy_web_socket = str2bool(data.proxy_web_socket)
data.proxy_http = str2bool(data.proxy_http)
data.proxy_https = str2bool(data.proxy_https)
data.proxy_https_redirect = str2bool(data.proxy_https_redirect)
data.port = service.obj['spec']['ports'][0]['port']
return data
if __name__ == "__main__":
config = KubeConfig.from_service_account()
api = HTTPClient(config)
services = []
for namespace in os.getenv('PROXYED_NAMESPACES', 'default').split(','):
services += Service.objects(api).filter(namespace=namespace, selector={'proxied': 'true'})
data = []
for service in services:
data.append(parse_service(service))
result = render(data)
with open('/etc/nginx/nginx.conf', 'w') as file:
file.write(result)
def run(self, resource, *args, **kwargs):
# TODO: clean on exceptions too
api = HTTPClient(KubeConfig.from_file('~/.kube/config'))
# handler = resource.db_obj.handler
command = args
items = self.get_volume_items(resource)
sync_transport = resource._bat_transport_sync
name = sync_transport.data_sufix
job_name = 'job' + name
# kubernetes api...
obj = {
'apiVersion': 'batch/v1',
'kind': 'Job',
'metadata': {'name': job_name},
'spec': {'template':
{'metadata': {
'name': 'cnts' + name
},
'spec': {
'containers': [
{'name': 'cnt' + name,
'image': 'solarproject/ansible:latest',
'command': command,
'volumeMounts': [
{'name': 'config-volume',
'mountPath': '/tmp'}
]}
],
'volumes': [
{'name': 'config-volume',
'configMap': {
'name': sync_transport.configmap_name,
'items': items
}}
],
'restartPolicy': 'OnFailure'
}}}}
self.job_obj = job_obj = Job(api, obj)
job_obj.create()
log.debug("Created JOB: %s", job_obj.name)
job_status = False
rc = 0
while True:
log.debug("Starting K8S job loop check")
time.sleep(1)
job_obj.reload()
job_status = job_obj.obj['status']
if job_status.get('active', 0) >= 1:
log.debug("Job is active")
# for now assuming that we have only one POD for JOB
pods = list(pykube.Pod.objects(api).filter(selector='job-name={}'.format(job_name)))
if pods:
pod = pods[0]
log.debug("Found pods for job")
rc, status = self._pod_status(pod)
if rc > 1:
log.debug("Container was restarted")
break
if status == 'Error':
log.debug("State is Error")
break
if job_status.get('succeeded', 0) >= 1:
log.debug("Job succeeded")
job_status = True
pods = list(pykube.Pod.objects(api).filter(selector='job-name={}'.format(job_name)))
pod = pods[0]
break
txt_logs = pod.get_logs()
log.debug("Output from POD: %s", txt_logs)
if job_status:
stdout = txt_logs
stderr = ''
else:
stdout = ''
stderr = txt_logs
self._clean_job(sync_transport.configmap_obj, self.job_obj)
return SolarTransportResult.from_tuple(rc, stdout, stderr)
