class MyFlowService(): def __init__(self): self.client = OperetoClient() self.input = self.client.input self.failed = False def run(self): try: def _end_fake_parent_process(parent_pid, child_process_pids=[]): if self.client.is_success(child_process_pids): self.client.stop_process(parent_pid, 'success') else: self.failed = True self.client.stop_process(parent_pid, 'failure') ## create a "fake" parent process parent_1_pid = self.client.create_process( 'fake_parent_process', title='First commands group') ## attach some services to it as child processes cmd_1_pid = self.client.create_process('shell_command', title='df -k', command='df -k', pflow_id=parent_1_pid) cmd_2_pid = self.client.create_process('shell_command', title='netstat -rn', command='netstat -rn', pflow_id=parent_1_pid) _end_fake_parent_process(parent_1_pid, [cmd_1_pid, cmd_2_pid]) time.sleep(10) ## create a second "fake" parent process parent_2_pid = self.client.create_process( 'fake_parent_process', title='Second commands group') ## attach some services to it as child processes cmd_3_pid = self.client.create_process('shell_command', title='df -k', command='df -k', pflow_id=parent_2_pid) cmd_4_pid = self.client.create_process('shell_command', title='netstat -rn', command='netstat -rn', pflow_id=parent_2_pid) _end_fake_parent_process(parent_2_pid, [cmd_3_pid, cmd_4_pid]) time.sleep(5) ## set final result if self.failed: return self.client.FAILURE return self.client.SUCCESS except: traceback.format_exc() return self.client.FAILURE
class ServiceRunner(ServiceTemplate): def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) def validate_input(self): input_scheme = { "type": "object", "properties": { "deployment_operation": { "enum": [ 'create_statefulset', 'modify_statefulset', 'delete_statefulset', 'update_worker_image' ] }, "deployment_name": { "type": ["null", "string"] }, "agent_java_config": { "type": "string", "minLength": 1 }, "agent_log_level": { "enum": ['info', 'warn', 'error', 'fatal', 'debug'] }, "worker_config": { "type": "string", "minLength": 1 }, "agent_properties": item_properties_scheme, "required": [ 'deployment_operation', 'agent_java_config', 'agent_log_level', 'worker_config', 'agent_properties' ], "additionalProperties": True } } validator = JsonSchemeValidator(self.input, input_scheme) validator.validate() if self.input['deployment_name'] == 'opereto-worker-node': raise OperetoRuntimeError( error= 'Deployment name is invalid, this name is used for Opereto standard workers. Please insert different name.' ) ## post_operations if self.input['post_operations']: validator = JsonSchemeValidator(self.input['post_operations'], included_services_scheme) validator.validate() def process(self): def _get_agent_names(): names = [] for count in range(self.worker_replicas): names.append(self.deployment_name + '-' + str(count)) return names def _modify_agent(agent_id): try: self.client.get_agent(agent_id) except OperetoClientError: self.client.create_agent( agent_id=agent_id, name=agent_id, description= 'This agent worker is part of {} worker stateful set.'. format(self.deployment_name)) time.sleep(2) agent_properties = self.input['agent_properties'] agent_properties.update({ 'opereto.shared': True, 'worker.label': self.deployment_name }) self.client.modify_agent_properties(agent_id, agent_properties) def _agents_status(online=True): while (True): ok = True for agent_id in _get_agent_names(): try: agent_attr = self.client.get_agent(agent_id) if agent_attr['online'] != online: ok = False break except OperetoClientError: pass if ok: break time.sleep(5) def _tearrdown_statefileset(): print 'Deleting worker stateful set..' self.deployment_info = self.kubernetes_api.delete_stateful_set( self.deployment_name) print 'Waiting that all worker pods will be offline (may take some time)..' _agents_status(online=False) if self.deployment_operation == 'create_statefulset': print 'Creating worker stateful set..' self.deployment_info = self.kubernetes_api.create_stateful_set( self.deployment_template) for agent_id in _get_agent_names(): _modify_agent(agent_id) print 'Waiting that all worker pods will be online (may take some time)..' _agents_status(online=True) self.deployment_info = self.kubernetes_api.get_stateful_set( self.deployment_name) print self.deployment_info.status ## run post install services post_install_pids = [] for agent_id in _get_agent_names(): for service in self.input['post_operations']: input = service.get('input') or {} agent_name = service.get('agents') or agent_id pid = self.client.create_process( service=service['service'], agent=agent_name, title=service.get('title'), **input) post_install_pids.append(pid) if post_install_pids and not self.client.is_success( post_install_pids): _tearrdown_statefileset() return self.client.FAILURE elif self.deployment_operation == 'modify_statefulset': print 'Modifying worker stateful set..' self.deployment_info = self.kubernetes_api.modify_stateful_set( self.deployment_name, self.deployment_template) for agent_id in _get_agent_names(): _modify_agent(agent_id) print 'Waiting that all worker pods will be online (may take some time)..' _agents_status(online=True) self.deployment_info = self.kubernetes_api.get_stateful_set( self.deployment_name) print self.deployment_info.status elif self.deployment_operation == 'delete_statefulset': _tearrdown_statefileset() else: raise OperetoRuntimeError(error='Invalid operation: {}'.format( self.deployment_operation)) return self.client.SUCCESS def setup(self): self.kubernetes_api = KubernetesAPI() self.deployment_name = self.input['deployment_name'] self.deployment_operation = self.input['deployment_operation'] self.deployment_info = {} self.deployment_template = self.input['deployment_template'] self.worker_replicas = self.deployment_template["spec"]["replicas"] if self.deployment_operation in [ 'create_statefulset', 'modify_statefulset' ]: if self.deployment_name: self.deployment_template["metadata"][ "name"] = self.deployment_name self.deployment_template["spec"]["template"]["spec"][ "containers"][0]["name"] = self.deployment_name + "-worker" try: self.deployment_template["spec"]["selector"][ "matchLabels"][ "app"] = self.deployment_name + "-cluster" except: pass try: self.deployment_template["spec"]["template"]['metadata'][ "labels"]["app"] = self.deployment_name + "-cluster" except: pass else: self.deployment_name = self.deployment_template["metadata"][ "name"] if not self.deployment_template["spec"]["template"]["spec"][ "containers"][0].get('env'): self.deployment_template["spec"]["template"]["spec"][ "containers"][0]['env'] = [] self.deployment_template["spec"]["template"]["spec"]["containers"][ 0]['env'] += [{ "name": "agent_name", "valueFrom": { "fieldRef": { "fieldPath": "metadata.name" } } }, { "name": "opereto_host", "value": self.input['opereto_host'] }, { "name": "opereto_user", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_USERNAME" } } }, { "name": "opereto_password", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_PASSWORD" } } }, { "name": "javaParams", "value": self.input['agent_java_config'], }, { "name": "log_level", "value": self.input['agent_log_level'] }] print 'Deployment template:\n{}'.format( json.dumps(self.deployment_template, indent=4)) def teardown(self): print self.deployment_info
class ServiceRunner(ServiceTemplate): def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) def validate_input(self): input_scheme = { "type": "object", "properties": { "pod_operation": { "enum": ['create_pod', 'delete_pod'] }, "pod_name": { "type": ["null", "string"] }, "pod_template": { "type": "object" }, "agent_java_config": { "type": "string", "minLength": 1 }, "agent_log_level": { "enum": ['info', 'warn', 'error', 'fatal', 'debug'] }, "worker_config": { "type": "string", "minLength": 1 }, "agent_properties": item_properties_scheme, "required": [ 'pod_operation', 'pod_template', 'agent_java_config', 'agent_log_level', 'worker_config', 'agent_properties' ], "additionalProperties": True } } validator = JsonSchemeValidator(self.input, input_scheme) validator.validate() if self.input['pod_operation'] == 'delete_pod' and not self.input.get( 'pod_name'): raise OperetoRuntimeError( error='Pod name must be provided for this operation') if self.input['pod_name'].startswith('opereto-worker-node'): raise OperetoRuntimeError( error= 'Pod name is invalid, this name is used for Opereto standard elastic workers. Please select a different name.' ) ## post_operations if self.input['post_operations']: validator = JsonSchemeValidator(self.input['post_operations'], included_services_scheme) validator.validate() def process(self): def _modify_agent(agent_id): self.client.create_agent( agent_id=agent_id, name=agent_id, description= 'This agent worker is part of {} worker stateful set.'.format( self.pod_name)) time.sleep(2) agent_properties = self.input['agent_properties'] agent_properties.update({ 'opereto.shared': True, 'worker.label': self.pod_name }) self.client.modify_agent_properties(agent_id, agent_properties) def _agents_status(online=True): while (True): try: agent_attr = self.client.get_agent(self.pod_name) if agent_attr['online'] == online: break except OperetoClientError: pass time.sleep(5) def _tearrdown_pod(): print 'Deleting worker pod..' self.pod_info = self.kubernetes_api.delete_pod(self.pod_name) print 'Waiting that worker pod will be offline (may take some time)..' _agents_status(online=False) print 'Agent {} is offline.'.format(self.pod_name) if self.pod_operation == 'create_pod': print 'Creating worker pod..' _modify_agent(self.pod_name) self.kubernetes_api.create_pod(self.pod_template) print 'Waiting that worker pod will be online (may take some time)..' _agents_status(online=True) print 'Agent {} is online.'.format(self.pod_name) self.pod_info = self.kubernetes_api.get_pod(self.pod_name) print self.pod_info.status ## run post install services for service in self.input['post_operations']: input = service.get('input') or {} agent = service.get('agents') or self.pod_name pid = self.client.create_process(service=service['service'], agent=agent, title=service.get('title'), **input) if not self.client.is_success(pid): _tearrdown_pod() return self.client.FAILURE elif self.pod_operation == 'delete_pod': _tearrdown_pod() else: raise OperetoRuntimeError( error='Invalid operation: {}'.format(self.pod_operation)) return self.client.SUCCESS def setup(self): self.kubernetes_api = KubernetesAPI() self.pod_name = self.input['pod_name'] self.pod_operation = self.input['pod_operation'] self.pod_info = {} self.pod_template = self.input['pod_template'] if self.pod_operation == 'create_pod': if self.pod_name: self.pod_template["metadata"]["name"] = self.pod_name self.pod_template["spec"]["containers"][0][ "name"] = self.pod_name + "-worker" else: self.pod_name = self.pod_template["metadata"]["name"] if not self.pod_template["spec"]["containers"][0].get('env'): self.pod_template["spec"]["containers"][0]['env'] = [] self.pod_template["spec"]["containers"][0]['env'] += [{ "name": "agent_name", "valueFrom": { "fieldRef": { "fieldPath": "metadata.name" } } }, { "name": "opereto_host", "value": self.input['opereto_host'] }, { "name": "opereto_user", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_USERNAME" } } }, { "name": "opereto_password", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_PASSWORD" } } }, { "name": "javaParams", "value": self.input['agent_java_config'], }, { "name": "log_level", "value": self.input['agent_log_level'] }] print 'Pod template:\n{}'.format( json.dumps(self.pod_template, indent=4)) def teardown(self): print self.pod_info