class ServiceRunner(ServiceTemplate):
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"deployment_operation": {
"enum": [
'create_statefulset', 'modify_statefulset',
'delete_statefulset', 'update_worker_image'
]
},
"deployment_name": {
"type": ["null", "string"]
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'deployment_operation', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['deployment_name'] == 'opereto-worker-node':
raise OperetoRuntimeError(
error=
'Deployment name is invalid, this name is used for Opereto standard workers. Please insert different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def process(self):
def _get_agent_names():
names = []
for count in range(self.worker_replicas):
names.append(self.deployment_name + '-' + str(count))
return names
def _modify_agent(agent_id):
try:
self.client.get_agent(agent_id)
except OperetoClientError:
self.client.create_agent(
agent_id=agent_id,
name=agent_id,
description=
'This agent worker is part of {} worker stateful set.'.
format(self.deployment_name))
time.sleep(2)
agent_properties = self.input['agent_properties']
agent_properties.update({
'opereto.shared': True,
'worker.label': self.deployment_name
})
self.client.modify_agent_properties(agent_id, agent_properties)
def _agents_status(online=True):
while (True):
ok = True
for agent_id in _get_agent_names():
try:
agent_attr = self.client.get_agent(agent_id)
if agent_attr['online'] != online:
ok = False
break
except OperetoClientError:
pass
if ok:
break
time.sleep(5)
def _tearrdown_statefileset():
print 'Deleting worker stateful set..'
self.deployment_info = self.kubernetes_api.delete_stateful_set(
self.deployment_name)
print 'Waiting that all worker pods will be offline (may take some time)..'
_agents_status(online=False)
if self.deployment_operation == 'create_statefulset':
print 'Creating worker stateful set..'
self.deployment_info = self.kubernetes_api.create_stateful_set(
self.deployment_template)
for agent_id in _get_agent_names():
_modify_agent(agent_id)
print 'Waiting that all worker pods will be online (may take some time)..'
_agents_status(online=True)
self.deployment_info = self.kubernetes_api.get_stateful_set(
self.deployment_name)
print self.deployment_info.status
## run post install services
post_install_pids = []
for agent_id in _get_agent_names():
for service in self.input['post_operations']:
input = service.get('input') or {}
agent_name = service.get('agents') or agent_id
pid = self.client.create_process(
service=service['service'],
agent=agent_name,
title=service.get('title'),
**input)
post_install_pids.append(pid)
if post_install_pids and not self.client.is_success(
post_install_pids):
_tearrdown_statefileset()
return self.client.FAILURE
elif self.deployment_operation == 'modify_statefulset':
print 'Modifying worker stateful set..'
self.deployment_info = self.kubernetes_api.modify_stateful_set(
self.deployment_name, self.deployment_template)
for agent_id in _get_agent_names():
_modify_agent(agent_id)
print 'Waiting that all worker pods will be online (may take some time)..'
_agents_status(online=True)
self.deployment_info = self.kubernetes_api.get_stateful_set(
self.deployment_name)
print self.deployment_info.status
elif self.deployment_operation == 'delete_statefulset':
_tearrdown_statefileset()
else:
raise OperetoRuntimeError(error='Invalid operation: {}'.format(
self.deployment_operation))
return self.client.SUCCESS
def setup(self):
self.kubernetes_api = KubernetesAPI()
self.deployment_name = self.input['deployment_name']
self.deployment_operation = self.input['deployment_operation']
self.deployment_info = {}
self.deployment_template = self.input['deployment_template']
self.worker_replicas = self.deployment_template["spec"]["replicas"]
if self.deployment_operation in [
'create_statefulset', 'modify_statefulset'
]:
if self.deployment_name:
self.deployment_template["metadata"][
"name"] = self.deployment_name
self.deployment_template["spec"]["template"]["spec"][
"containers"][0]["name"] = self.deployment_name + "-worker"
try:
self.deployment_template["spec"]["selector"][
"matchLabels"][
"app"] = self.deployment_name + "-cluster"
except:
pass
try:
self.deployment_template["spec"]["template"]['metadata'][
"labels"]["app"] = self.deployment_name + "-cluster"
except:
pass
else:
self.deployment_name = self.deployment_template["metadata"][
"name"]
if not self.deployment_template["spec"]["template"]["spec"][
"containers"][0].get('env'):
self.deployment_template["spec"]["template"]["spec"][
"containers"][0]['env'] = []
self.deployment_template["spec"]["template"]["spec"]["containers"][
0]['env'] += [{
"name": "agent_name",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}, {
"name": "opereto_host",
"value": self.input['opereto_host']
}, {
"name": "opereto_user",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_USERNAME"
}
}
}, {
"name": "opereto_password",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_PASSWORD"
}
}
}, {
"name": "javaParams",
"value": self.input['agent_java_config'],
}, {
"name": "log_level",
"value": self.input['agent_log_level']
}]
print 'Deployment template:\n{}'.format(
json.dumps(self.deployment_template, indent=4))
def teardown(self):
print self.deployment_info
class ServiceRunner(ServiceTemplate):
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"pod_operation": {
"enum": ['create_pod', 'delete_pod']
},
"pod_name": {
"type": ["null", "string"]
},
"pod_template": {
"type": "object"
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'pod_operation', 'pod_template', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['pod_operation'] == 'delete_pod' and not self.input.get(
'pod_name'):
raise OperetoRuntimeError(
error='Pod name must be provided for this operation')
if self.input['pod_name'].startswith('opereto-worker-node'):
raise OperetoRuntimeError(
error=
'Pod name is invalid, this name is used for Opereto standard elastic workers. Please select a different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def process(self):
def _modify_agent(agent_id):
self.client.create_agent(
agent_id=agent_id,
name=agent_id,
description=
'This agent worker is part of {} worker stateful set.'.format(
self.pod_name))
time.sleep(2)
agent_properties = self.input['agent_properties']
agent_properties.update({
'opereto.shared': True,
'worker.label': self.pod_name
})
self.client.modify_agent_properties(agent_id, agent_properties)
def _agents_status(online=True):
while (True):
try:
agent_attr = self.client.get_agent(self.pod_name)
if agent_attr['online'] == online:
break
except OperetoClientError:
pass
time.sleep(5)
def _tearrdown_pod():
print 'Deleting worker pod..'
self.pod_info = self.kubernetes_api.delete_pod(self.pod_name)
print 'Waiting that worker pod will be offline (may take some time)..'
_agents_status(online=False)
print 'Agent {} is offline.'.format(self.pod_name)
if self.pod_operation == 'create_pod':
print 'Creating worker pod..'
_modify_agent(self.pod_name)
self.kubernetes_api.create_pod(self.pod_template)
print 'Waiting that worker pod will be online (may take some time)..'
_agents_status(online=True)
print 'Agent {} is online.'.format(self.pod_name)
self.pod_info = self.kubernetes_api.get_pod(self.pod_name)
print self.pod_info.status
## run post install services
for service in self.input['post_operations']:
input = service.get('input') or {}
agent = service.get('agents') or self.pod_name
pid = self.client.create_process(service=service['service'],
agent=agent,
title=service.get('title'),
**input)
if not self.client.is_success(pid):
_tearrdown_pod()
return self.client.FAILURE
elif self.pod_operation == 'delete_pod':
_tearrdown_pod()
else:
raise OperetoRuntimeError(
error='Invalid operation: {}'.format(self.pod_operation))
return self.client.SUCCESS
def setup(self):
self.kubernetes_api = KubernetesAPI()
self.pod_name = self.input['pod_name']
self.pod_operation = self.input['pod_operation']
self.pod_info = {}
self.pod_template = self.input['pod_template']
if self.pod_operation == 'create_pod':
if self.pod_name:
self.pod_template["metadata"]["name"] = self.pod_name
self.pod_template["spec"]["containers"][0][
"name"] = self.pod_name + "-worker"
else:
self.pod_name = self.pod_template["metadata"]["name"]
if not self.pod_template["spec"]["containers"][0].get('env'):
self.pod_template["spec"]["containers"][0]['env'] = []
self.pod_template["spec"]["containers"][0]['env'] += [{
"name": "agent_name",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}, {
"name":
"opereto_host",
"value":
self.input['opereto_host']
}, {
"name": "opereto_user",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_USERNAME"
}
}
}, {
"name": "opereto_password",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_PASSWORD"
}
}
}, {
"name":
"javaParams",
"value":
self.input['agent_java_config'],
}, {
"name":
"log_level",
"value":
self.input['agent_log_level']
}]
print 'Pod template:\n{}'.format(
json.dumps(self.pod_template, indent=4))
def teardown(self):
print self.pod_info