class MyService(): def __init__(self): self.client = OperetoClient() self.input = self.client.input print self.input def run(self): try: ## print the input parameter print 'The input parameters passed: %s' % str( self.input['my_input_param']) ## modify output parameter with input one self.client.modify_process_property('my_output_param', self.input['my_input_param']) ## verify that input text started with Hello World if not self.input['my_input_param'].startswith('Hello World'): print >> sys.stderr, 'Input does not start with Hello World' return self.client.FAILURE return self.client.SUCCESS except: traceback.format_exc() return self.client.FAILURE
def get_opereto_client(): if opereto_credentials_json.get('opereto_password'): client = OperetoClient(opereto_host=OPERETO_HOST, opereto_user=opereto_credentials_json['opereto_user'], opereto_password=opereto_credentials_json['opereto_password']) return client else: client = OperetoClient(opereto_host=OPERETO_HOST, opereto_token=opereto_credentials_json['opereto_token']) return client
class ProcessLogScheduler(): """ from time import sleep with ProcessLogScheduler(1) as p: p.write('eAsEFPGfJun', [{'text': 'my log entry 1', 'level': 'INFO'}]) p.write('eAsEFPGfJun', [{'text': 'my log entry 2', 'level': 'INFO'}]) p.write('53cqHRdu4Cr', [{'text': 'some log text..', 'level': 'ERROR'}]) sleep(5) # your long-running job goes here... """ def __init__(self, interval=5): self.buffer = { 0: {}, 1: {} } self.active_buffer = 0 self._timer = None self.client = OperetoClient() self.is_running = False self.interval = interval self.start() def __enter__(self): return self def __exit__(self, type, value, traceback): self.flush_logs() def _switch_buffer(self): active_buffer_sn = self.active_buffer self.active_buffer = abs(int(self.active_buffer) - 1) logs = self.buffer[active_buffer_sn] self.buffer[active_buffer_sn]={} return logs def flush_logs(self, *args): self.is_running = False self.start() for pid, entries in self._switch_buffer().items(): self.client.send_process_log(pid, entries) def start(self): if not self.is_running: self._timer = Timer(self.interval, self.flush_logs) self._timer.daemon = True self._timer.start() self.is_running = True def stop(self): self._timer.cancel() self.is_running = False def write(self, pid, log_entries=[]): if not pid in self.buffer[self.active_buffer]: self.buffer[self.active_buffer][pid]=[] self.buffer[self.active_buffer][pid]+=log_entries
def remove_opereto_lib(): try: c = OperetoClient() c.modify_agent_property(c.input['opereto_agent'], 'opereto.worker', False) return c.SUCCESS except Exception as e: print(e) return 2
def __init__(self, interval=5): self.buffer = { 0: {}, 1: {} } self.active_buffer = 0 self._timer = None self.client = OperetoClient() self.is_running = False self.interval = interval self.start()
def get_opereto_client(): try: OPERETO_AUTH_TOKEN = opereto_credentials_json['opereto_auth_token'] client = OperetoClient(opereto_host=OPERETO_HOST, opereto_auth_token=OPERETO_AUTH_TOKEN) except KeyError: opereto_user = opereto_credentials_json['opereto_user'] opereto_password = opereto_credentials_json['opereto_password'] client = OperetoClient(opereto_host=OPERETO_HOST, opereto_user=opereto_user, opereto_password=opereto_password) return client
def __init__(self, **kwargs): self.client = OperetoClient() self.input = self.client.input self.exitcode = 0 if kwargs: self.input.update(kwargs)
def __init__(self, **kwargs): self.client = OperetoClient() self.input = self.client.input self.state_file = os.path.join(self.input['opereto_workspace'], '_opereto_service_state.json') self.exitcode = 0 if kwargs: self.input.update(kwargs)
class MyService: def __init__(self): self.client = OperetoClient() self.input = self.client.input def run(self): try: ## print the input parameter print "The input parameters passed: %s" % str(self.input["my_input_param"]) ## modify output parameter with input one self.client.modify_process_property("my_output_param", self.input["my_input_param"]) ## verify that input text started with Hello World if not self.input["my_input_param"].startswith("Hello World"): print >> sys.stderr, "Input does not start with Hello World" return self.client.FAILURE return self.client.SUCCESS except: traceback.format_exc() return self.client.FAILURE
import os from pyopereto.client import OperetoClient from pyopereto.helpers.packages import OperetoAwsS3PackagesManager package_directory = os.path.dirname(os.getcwd()) ## set the AWS S3 bucket name, access key and secret key client = OperetoClient() opereto_packages_repo_ak = client.input['opereto_packages_repo_ak'] opereto_packages_repo_sk = client.input['opereto_packages_repo_sk'] opereto_packages_bucket_name = client.input['opereto_packages_bucket_name'] if __name__ == "__main__": obj = OperetoAwsS3PackagesManager(package_directory, opereto_packages_bucket_name, opereto_packages_repo_ak, opereto_packages_repo_sk, 'opereto-gcp-services') obj.deploy()
def install_opereto_lib(): LIB_DIR = os.path.join(os.environ['opereto_workspace'], 'opereto') VIRT_ENV_DIR = os.path.join(os.environ['opereto_home'], 'operetovenv') per_os_module_to_install = { 'rhel7.2': { 'paramiko': 'paramiko==2.1.2' } } module_to_install = { 'yaml' : 'pyyaml==3.13', 'pyopereto': 'pyopereto==1.0.55', 'requests': 'requests==2.19.1', 'boto': 'boto==2.49.0', 'paramiko': 'paramiko==2.4.1', 'six': 'six==1.11.0', 'sh': 'sh==1.12.14', 'werkzeug': 'werkzeug==0.14.1', 'psutil': 'psutil==0.7.0', 'dateutil': 'python-dateutil==2.7.3', 'jsonschema': 'jsonschema==2.6.0', 'faker_schema': 'faker-schema==0.1.4', 'docker': 'docker==3.1.4', 'PIL': 'Pillow==2.9.0', 'boto3': 'boto3==1.7.39', 'httplib2': 'httplib2==0.11.3', 'oauth2client': 'oauth2client==4.1.2', 'apiclient': 'google-api-python-client==1.7.4', 'junitparser': 'junitparser==1.2.2', 'kubernetes':'kubernetes==5.0.0' } def get_current_os(): (name, version,id) = platform.linux_distribution() return (name, version,id) def is_ubuntu(): (name, version,id) = get_current_os() if name == 'Ubuntu': return True return False def is_windows(): if platform.system() == 'Windows': return True return False def _remove_dir(dir): if os.path.exists(dir): if is_windows(): _local("rmdir /s /q %s"%dir) else: _local("sudo rm -rf %s"%dir) def _create_dir_if_not_exists(dir): if not os.path.exists(dir): if is_windows(): _local("mkdir %s" % dir) else: _local("sudo mkdir -p %s" % dir) _local('sudo chmod -R 777 ' + dir) def _copy_opereto_venv_contents(dir): if is_windows(): WIN_LIB_DIR = LIB_DIR.replace("/","\\") _local("xcopy %s %s /q /i /e /Y" % (WIN_LIB_DIR, os.path.join(dir, 'opereto', 'venv'))) else: _local("cp -rf %s %s" % (LIB_DIR, os.path.join(dir, 'opereto'))) def _local(cmd, ignore=False): global FAILURE print cmd ret = os.system(cmd) if int(ret)!=0 and not ignore: print 'Command ended with exit code: {}'.format(int(ret)) FAILURE=True return ret def _prepare_new_virtual_env(directory): os_name=None (name, version,id) = get_current_os() print 'Current OS: {} {}, {}'.format(name, version,id) if name=='Red Hat Enterprise Linux Server' and version=='7.2': os_name='rhel7.2' if os_name: module_to_install.update(per_os_module_to_install[os_name]) if is_windows(): _local('cd %s && virtualenv venv' % (directory), ignore=False) for import_name, module in module_to_install.items(): _local('cd %s && pip install %s' % (os.path.join(directory,'venv', 'Scripts'), module), ignore=False) if _local('cd %s && python -c "import %s"' % (os.path.join(directory, 'venv', 'Scripts'), import_name)) != 0: print >> sys.stderr, 'Python module [%s] is not installed.' % module else: _local('cd %s && virtualenv venv && . venv/bin/activate && pip install --upgrade pip ; deactivate'%(directory),ignore=False) for import_name, module in module_to_install.items(): _local('cd %s && . venv/bin/activate && pip install %s && deactivate'%(directory, module),ignore=True) if _local('cd %s && . venv/bin/activate && python -c "import %s" && deactivate'%(directory, import_name))!=0: print >> sys.stderr, 'Python module [%s] is not installed.'%module def create_virtual_rlauto_env(current_version_dir): _remove_dir(current_version_dir) _create_dir_if_not_exists(current_version_dir) _copy_opereto_venv_contents(current_version_dir) _prepare_new_virtual_env(current_version_dir) if not is_windows(): _local('sudo chmod 777 -R %s'%current_version_dir) _local("sudo su -c 'echo \"export PYTHONPATH=%s\" >> %s'"%(current_version_dir,os.path.join(current_version_dir,'venv/bin/activate'))) try: release=os.environ.get('opereto_service_version') ## check prerequisites if sys.version_info<(2,7): print >> sys.stderr,'Opereto microservices lib requires python 2.7 or higher.' return 2 if is_ubuntu(): (name, version,id) = get_current_os() install_list = 'sudo apt-get install -qy python-six curl python-setuptools gcc build-essential python-dev python-pip libffi-dev libssl-dev' _local(install_list,ignore=False) _local('sudo pip install virtualenv', ignore=False) elif is_windows(): _local('pip install virtualenv', ignore=False) _local('pip install pyopereto') else: _local('sudo yum install -y python-virtualenv python-setuptools gcc libffi python-devel openssl-devel') _local('easy_install pip') if not is_windows(): _local('sudo pip install pyopereto') current_version_dir= os.path.join(VIRT_ENV_DIR,release) create_virtual_rlauto_env(current_version_dir) if FAILURE: return 3 else: if os.environ.get('opereto_agent'): from pyopereto.client import OperetoClient c = OperetoClient() if c.input['standard_opereto_worker']: c.modify_agent_property(c.input['opereto_agent'], 'opereto.worker', True) return 0 except Exception, e: print traceback.format_exc() return 2
class ServiceRunner(ServiceTemplate): def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) def validate_input(self): input_scheme = { "type": "object", "properties": { "pod_operation": { "enum": ['create_pod', 'delete_pod'] }, "pod_name": { "type": ["null", "string"] }, "pod_template": { "type": "object" }, "agent_java_config": { "type": "string", "minLength": 1 }, "agent_log_level": { "enum": ['info', 'warn', 'error', 'fatal', 'debug'] }, "worker_config": { "type": "string", "minLength": 1 }, "agent_properties": item_properties_scheme, "required": [ 'pod_operation', 'pod_template', 'agent_java_config', 'agent_log_level', 'worker_config', 'agent_properties' ], "additionalProperties": True } } validator = JsonSchemeValidator(self.input, input_scheme) validator.validate() if self.input['pod_operation'] == 'delete_pod' and not self.input.get( 'pod_name'): raise OperetoRuntimeError( error='Pod name must be provided for this operation') if self.input['pod_name'].startswith('opereto-worker-node'): raise OperetoRuntimeError( error= 'Pod name is invalid, this name is used for Opereto standard elastic workers. Please select a different name.' ) ## post_operations if self.input['post_operations']: validator = JsonSchemeValidator(self.input['post_operations'], included_services_scheme) validator.validate() def process(self): def _modify_agent(agent_id): self.client.create_agent( agent_id=agent_id, name=agent_id, description= 'This agent worker is part of {} worker stateful set.'.format( self.pod_name)) time.sleep(2) agent_properties = self.input['agent_properties'] agent_properties.update({ 'opereto.shared': True, 'worker.label': self.pod_name }) self.client.modify_agent_properties(agent_id, agent_properties) def _agents_status(online=True): while (True): try: agent_attr = self.client.get_agent(self.pod_name) if agent_attr['online'] == online: break except OperetoClientError: pass time.sleep(5) def _tearrdown_pod(): print 'Deleting worker pod..' self.pod_info = self.kubernetes_api.delete_pod(self.pod_name) print 'Waiting that worker pod will be offline (may take some time)..' _agents_status(online=False) print 'Agent {} is offline.'.format(self.pod_name) if self.pod_operation == 'create_pod': print 'Creating worker pod..' _modify_agent(self.pod_name) self.kubernetes_api.create_pod(self.pod_template) print 'Waiting that worker pod will be online (may take some time)..' _agents_status(online=True) print 'Agent {} is online.'.format(self.pod_name) self.pod_info = self.kubernetes_api.get_pod(self.pod_name) print self.pod_info.status ## run post install services for service in self.input['post_operations']: input = service.get('input') or {} agent = service.get('agents') or self.pod_name pid = self.client.create_process(service=service['service'], agent=agent, title=service.get('title'), **input) if not self.client.is_success(pid): _tearrdown_pod() return self.client.FAILURE elif self.pod_operation == 'delete_pod': _tearrdown_pod() else: raise OperetoRuntimeError( error='Invalid operation: {}'.format(self.pod_operation)) return self.client.SUCCESS def setup(self): self.kubernetes_api = KubernetesAPI() self.pod_name = self.input['pod_name'] self.pod_operation = self.input['pod_operation'] self.pod_info = {} self.pod_template = self.input['pod_template'] if self.pod_operation == 'create_pod': if self.pod_name: self.pod_template["metadata"]["name"] = self.pod_name self.pod_template["spec"]["containers"][0][ "name"] = self.pod_name + "-worker" else: self.pod_name = self.pod_template["metadata"]["name"] if not self.pod_template["spec"]["containers"][0].get('env'): self.pod_template["spec"]["containers"][0]['env'] = [] self.pod_template["spec"]["containers"][0]['env'] += [{ "name": "agent_name", "valueFrom": { "fieldRef": { "fieldPath": "metadata.name" } } }, { "name": "opereto_host", "value": self.input['opereto_host'] }, { "name": "opereto_user", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_USERNAME" } } }, { "name": "opereto_password", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_PASSWORD" } } }, { "name": "javaParams", "value": self.input['agent_java_config'], }, { "name": "log_level", "value": self.input['agent_log_level'] }] print 'Pod template:\n{}'.format( json.dumps(self.pod_template, indent=4)) def teardown(self): print self.pod_info
#!/usr/bin/python from pyopereto.client import OperetoClient from optparse import OptionParser def parse_options(): usage = "%prog -s service [-v version]" parser = OptionParser(usage=usage) parser.add_option("-s", "--service", dest="service", help="service name.") parser.add_option("-v", "--version", dest="version", help="delete a specific service production version, for instance: 1.1.0-12.") (options, args) = parser.parse_args() if not options.service: parser.error('service must be provided.') return (options, args) if __name__ == "__main__": (options, args) = parse_options() client = OperetoClient() operations_mode = 'development' if options.version: operations_mode = 'production' version=options.version if not version: version='default' client.delete_service_version(service_id=options.service, service_version=version, mode=operations_mode) if operations_mode=='production': print 'Version [%s] of service [%s] has been deleted'%(options.service, options.version) else: print 'Development version of service [%s] has been deleted'%options.service
def get_opereto_client(): client = OperetoClient( opereto_host=opereto_credentials_json['opereto_host'], opereto_user=opereto_credentials_json['opereto_user'], opereto_password=opereto_credentials_json['opereto_password']) return client
def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) self.sflow_id = self.input['opereto_source_flow_id'] self.remove_test_results_dir = False self.op_state = self._get_state() or {}
def __init__(self): self.client = OperetoClient() self.input = self.client.input self.failed = False
def install_opereto_lib(): LIB_DIR = os.path.join(os.environ['opereto_workspace'], 'opereto') VIRT_ENV_DIR = os.path.join(os.environ['opereto_home'], 'operetovenv') module_to_install = { 'boto3': 'boto3==1.20.6', 'yaml': 'pyyaml==6.0', 'pyopereto': 'pyopereto==1.0.132', 'kubernetes': 'kubernetes==19.15.0', 'requests': 'requests==2.26.0', 'paramiko': 'paramiko==2.8.0', 'six': 'six==1.16.0', 'sh': 'sh==1.14.2', 'werkzeug': 'werkzeug==2.0.2', 'psutil': 'psutil==5.8.0', 'dateutil': 'python-dateutil==2.8.2', 'jsonschema': 'jsonschema==2.6.0', 'faker_schema': 'faker-schema==0.1.4', 'docker': 'docker==5.0.3', 'PIL': 'Pillow==8.4.0', 'httplib2': 'httplib2==0.20.2', 'oauth2client': 'oauth2client==4.1.3', 'apiclient': 'google-api-python-client==2.30.0', 'junitparser': 'junitparser==2.1.1' } def get_pip_command(): if version == '20.04': return 'pip3' return 'pip3.8' def get_virtenv_cmd(): return 'virtualenv' def get_current_os(): out = subprocess.check_output('cat /etc/*-release', shell=True).decode('utf-8') name = re.search('NAME="([A-Za-z0-9\s]*)"', out) version = re.search('VERSION_ID=\"([\d\.\-]*)\"', out) os_id = re.search('ID=([A-Za-z0-9\s\"]*)\n', out) if name and version and os_id: return name.group(1), version.group(1), os_id.group(1).strip('"') else: raise Exception( f'cant find OS details (NAME, VERSION_ID, ID), check full os-release output:\n{out}' ) def _remove_dir(dir): if os.path.exists(dir): _local("sudo rm -rf {}".format(dir)) def _create_dir_if_not_exists(dir): if not os.path.exists(dir): _local("sudo mkdir -p {}".format(dir)) _local('sudo chmod -R 777 ' + dir) def _copy_opereto_venv_contents(dir): _local("cp -rf {} {}".format(LIB_DIR, os.path.join(dir, 'opereto'))) def _local(cmd, ignore=False): global FAILURE print(cmd) ret = os.system(cmd) if int(ret) != 0 and not ignore: print('Command ended with exit code: {}'.format(int(ret))) FAILURE = True return ret def _prepare_new_virtual_env(directory): (name, version, id) = get_current_os() print('Current OS: {} {}, {}'.format(name, version, id)) _local( 'cd {} && {} venv -p python3.6 && . venv/bin/activate && pip install --upgrade pip ; deactivate' .format(directory, get_virtenv_cmd()), ignore=False) for import_name, module in module_to_install.items(): _local( 'cd {} && . venv/bin/activate && pip install {} && deactivate'. format(directory, module), ignore=True) if _local( 'cd {} && . venv/bin/activate && python -c "import {}" && deactivate' .format(directory, import_name)) != 0: sys.stderr.write( 'Python module [{}] is not installed.'.format(module)) def create_virtual_env(current_version_dir): _remove_dir(current_version_dir) _create_dir_if_not_exists(current_version_dir) _copy_opereto_venv_contents(current_version_dir) _prepare_new_virtual_env(current_version_dir) _local('sudo chmod 777 -R {}'.format(current_version_dir)) _local("sudo su -c 'echo \"export PYTHONPATH={}\" >> {}'".format( current_version_dir, os.path.join(current_version_dir, 'venv/bin/activate'))) try: release = os.environ.get('opereto_service_version') (name, version, id) = get_current_os() if sys.version_info < (3, 6): sys.stderr.write( 'Opereto microservices lib requires python 3.6 or higher.') return 2 _local( 'export DEBIAN_FRONTEND=noninteractive ; sudo -E apt-get update') install_list = 'sudo apt-get install -qy python-six curl python-setuptools gcc build-essential python3.6-dev libffi-dev libssl-dev' _local(install_list, ignore=False) _local('curl https://bootstrap.pypa.io/get-pip.py | sudo -H python3.8') _local( 'export LC_ALL=\"en_US.UTF-8\" ; export LC_CTYPE=\"en_US.UTF-8\" ; sudo {} install virtualenv==16.0.0' .format(get_pip_command()), ignore=False) current_version_dir = os.path.join(VIRT_ENV_DIR, release) create_virtual_env(current_version_dir) if FAILURE: return 3 try: _local('{} install --upgrade pyopereto==1.0.124'.format( get_pip_command())) if os.environ.get('opereto_agent'): from pyopereto.client import OperetoClient c = OperetoClient() if c.input['standard_opereto_worker']: c.modify_agent_property(c.input['opereto_agent'], 'opereto.worker', True) except Exception as e: print(e) return 0 except Exception as e: print(traceback.format_exc()) return 2
class ServiceRunner(ServiceTemplate): def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) def validate_input(self): input_scheme = { "type": "object", "properties": { "deployment_operation": { "enum": [ 'create_statefulset', 'modify_statefulset', 'delete_statefulset', 'update_worker_image' ] }, "deployment_name": { "type": ["null", "string"] }, "agent_java_config": { "type": "string", "minLength": 1 }, "agent_log_level": { "enum": ['info', 'warn', 'error', 'fatal', 'debug'] }, "worker_config": { "type": "string", "minLength": 1 }, "agent_properties": item_properties_scheme, "required": [ 'deployment_operation', 'agent_java_config', 'agent_log_level', 'worker_config', 'agent_properties' ], "additionalProperties": True } } validator = JsonSchemeValidator(self.input, input_scheme) validator.validate() if self.input['deployment_name'] == 'opereto-worker-node': raise OperetoRuntimeError( error= 'Deployment name is invalid, this name is used for Opereto standard workers. Please insert different name.' ) ## post_operations if self.input['post_operations']: validator = JsonSchemeValidator(self.input['post_operations'], included_services_scheme) validator.validate() def process(self): def _get_agent_names(): names = [] for count in range(self.worker_replicas): names.append(self.deployment_name + '-' + str(count)) return names def _modify_agent(agent_id): try: self.client.get_agent(agent_id) except OperetoClientError: self.client.create_agent( agent_id=agent_id, name=agent_id, description= 'This agent worker is part of {} worker stateful set.'. format(self.deployment_name)) time.sleep(2) agent_properties = self.input['agent_properties'] agent_properties.update({ 'opereto.shared': True, 'worker.label': self.deployment_name }) self.client.modify_agent_properties(agent_id, agent_properties) def _agents_status(online=True): while (True): ok = True for agent_id in _get_agent_names(): try: agent_attr = self.client.get_agent(agent_id) if agent_attr['online'] != online: ok = False break except OperetoClientError: pass if ok: break time.sleep(5) def _tearrdown_statefileset(): print 'Deleting worker stateful set..' self.deployment_info = self.kubernetes_api.delete_stateful_set( self.deployment_name) print 'Waiting that all worker pods will be offline (may take some time)..' _agents_status(online=False) if self.deployment_operation == 'create_statefulset': print 'Creating worker stateful set..' self.deployment_info = self.kubernetes_api.create_stateful_set( self.deployment_template) for agent_id in _get_agent_names(): _modify_agent(agent_id) print 'Waiting that all worker pods will be online (may take some time)..' _agents_status(online=True) self.deployment_info = self.kubernetes_api.get_stateful_set( self.deployment_name) print self.deployment_info.status ## run post install services post_install_pids = [] for agent_id in _get_agent_names(): for service in self.input['post_operations']: input = service.get('input') or {} agent_name = service.get('agents') or agent_id pid = self.client.create_process( service=service['service'], agent=agent_name, title=service.get('title'), **input) post_install_pids.append(pid) if post_install_pids and not self.client.is_success( post_install_pids): _tearrdown_statefileset() return self.client.FAILURE elif self.deployment_operation == 'modify_statefulset': print 'Modifying worker stateful set..' self.deployment_info = self.kubernetes_api.modify_stateful_set( self.deployment_name, self.deployment_template) for agent_id in _get_agent_names(): _modify_agent(agent_id) print 'Waiting that all worker pods will be online (may take some time)..' _agents_status(online=True) self.deployment_info = self.kubernetes_api.get_stateful_set( self.deployment_name) print self.deployment_info.status elif self.deployment_operation == 'delete_statefulset': _tearrdown_statefileset() else: raise OperetoRuntimeError(error='Invalid operation: {}'.format( self.deployment_operation)) return self.client.SUCCESS def setup(self): self.kubernetes_api = KubernetesAPI() self.deployment_name = self.input['deployment_name'] self.deployment_operation = self.input['deployment_operation'] self.deployment_info = {} self.deployment_template = self.input['deployment_template'] self.worker_replicas = self.deployment_template["spec"]["replicas"] if self.deployment_operation in [ 'create_statefulset', 'modify_statefulset' ]: if self.deployment_name: self.deployment_template["metadata"][ "name"] = self.deployment_name self.deployment_template["spec"]["template"]["spec"][ "containers"][0]["name"] = self.deployment_name + "-worker" try: self.deployment_template["spec"]["selector"][ "matchLabels"][ "app"] = self.deployment_name + "-cluster" except: pass try: self.deployment_template["spec"]["template"]['metadata'][ "labels"]["app"] = self.deployment_name + "-cluster" except: pass else: self.deployment_name = self.deployment_template["metadata"][ "name"] if not self.deployment_template["spec"]["template"]["spec"][ "containers"][0].get('env'): self.deployment_template["spec"]["template"]["spec"][ "containers"][0]['env'] = [] self.deployment_template["spec"]["template"]["spec"]["containers"][ 0]['env'] += [{ "name": "agent_name", "valueFrom": { "fieldRef": { "fieldPath": "metadata.name" } } }, { "name": "opereto_host", "value": self.input['opereto_host'] }, { "name": "opereto_user", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_USERNAME" } } }, { "name": "opereto_password", "valueFrom": { "secretKeyRef": { "name": self.input['worker_config'], "key": "OPERETO_PASSWORD" } } }, { "name": "javaParams", "value": self.input['agent_java_config'], }, { "name": "log_level", "value": self.input['agent_log_level'] }] print 'Deployment template:\n{}'.format( json.dumps(self.deployment_template, indent=4)) def teardown(self): print self.deployment_info
def __init__(self): self.client = OperetoClient() self.input = self.client.input print self.input
def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) self._print_step_title('Start opereto test listener..') self.sflow_id = self.input['opereto_source_flow_id'] self.remove_test_results_dir=False
class ServiceRunner(ServiceTemplate): def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs) self.sflow_id = self.input['opereto_source_flow_id'] self.remove_test_results_dir = False self.op_state = self._get_state() or {} def validate_input(self): input_scheme = { "type": "object", "properties": { "test_results_path": { "type": "string", "minLength": 1 }, "parent_pid": { "type": "string" }, "listener_frequency": { "type": "integer", "minValue": 1 }, "debug_mode": { "type": "boolean" } }, "required": ['listener_frequency', 'test_results_path'], "additionalProperties": True } validator = JsonSchemeValidator(self.input, input_scheme) validator.validate() self.parent_pid = self.input['parent_pid'] or self.input['pid'] self.test_results_dir = self.input['test_results_path'] self.debug_mode = self.input['debug_mode'] self.result_keys = process_result_keys self.status_keys = process_status_keys self.tests_json_scheme = { "type": "object", "properties": { "test_suite": { "type": "object", "properties": { "links": { "type": "array" }, "status": { "enum": self.status_keys } } }, "test_records": { "type": "array", "items": [{ "type": "object", "properties": { "testname": default_entity_name_scheme, "status": { "enum": self.status_keys }, "title": default_entity_name_scheme, "links": { "type": "array", "items": [{ "type": "object", "properties": { "url": { "type": "string" }, "name": { "type": "string" } } }] } }, "required": ['testname', 'status'], "additionalProperties": True }] } }, "additionalProperties": True } self.end_of_test_suite = None self.test_data = {} self.suite_links = [] self._state = {} def _print_test_link(self, link): print( '[OPERETO_HTML]<br><a href="{}"><font style="color: #222; font-weight: 600; font-size: 13px;">{}</font></a>' .format(link['url'], link['name'])) def _append_to_process_log(self, pid, ppid, loglines, log_level='info'): log_request_data = { 'sflow_id': self.sflow_id, 'pflow_id': ppid, 'agent_id': self.input['opereto_agent'], 'product_id': self.input['opereto_product_id'], 'data': [] } count = 1 for line in loglines: try: millis = int(round(time.time() * 1000)) + count log_request_data['data'].append({ 'level': log_level, 'text': line.strip(), 'timestamp': millis }) except Exception as e: print(e) count += 1 self.client._call_rest_api( 'post', '/processes/{}/log'.format(pid), data=log_request_data, error='Failed to update test log (test pid = {})'.format(pid)) def _modify_record(self, test_record): testname = test_record['testname'] test_input = test_record.get('test_input') or {} title = test_record.get('title') or testname status = test_record['status'] test_links = test_record.get('links') or [] test_ppid = test_record.get('ppid') or self.parent_pid test_pid = test_record.get('pid') if testname not in self._state: if not test_pid: test_pid = self.client.create_process( 'opereto_test_listener_record', testname=testname, title=title, test_input=test_input, test_runner_id=test_ppid, pflow_id=test_ppid) self.client.wait_to_start(test_pid) self._state[testname] = { 'ppid': test_ppid, 'pid': test_pid, 'status': 'in_process', 'title': title, 'test_output_md5': '', 'summary_md5': '', 'last_log_line': 1 } else: test_pid = self._state[testname]['pid'] if self._state[testname]['status'] not in self.result_keys: if title != self._state[testname]['title']: ### TBD: add title change API call self._state[testname]['title'] = title results_dir = os.path.join(self.test_results_dir, testname) if os.path.exists(results_dir): output_json_file = os.path.join(results_dir, 'output.json') log_file = os.path.join(results_dir, 'stdout.log') summary_file = os.path.join(results_dir, 'summary.txt') if os.path.exists(output_json_file): output_json_md5 = get_file_md5sum(output_json_file) if output_json_md5 != self._state[testname][ 'test_output_md5']: with open(output_json_file, 'r') as of: output_json = json.load(of) self.client.modify_process_property('test_output', output_json, pid=test_pid) self._state[testname][ 'test_output_md5'] = output_json_md5 if os.path.exists(summary_file): summary_md5 = get_file_md5sum(summary_file) if summary_md5 != self._state[testname]['summary_md5']: with open(summary_file, 'r') as sf: summary = sf.read() self.client.modify_process_summary( test_pid, summary) self._state[testname]['summary_md5'] = summary_md5 if os.path.exists(log_file): with open(log_file, 'r') as lf: count = 1 loglines = [] for line in lf.readlines(): if count >= self._state[testname]['last_log_line']: if count > MAX_LOG_LINES_PER_PROCESS: message = 'Test log is too long. Please save test log in remote storage and add a link to it in Opereto log. See service info to learn how to add links to your tests.json file.' loglines.append( '[OPERETO_HTML]<br><br><font style="width: 800px; padding: 15px; color: #222; font-weight: 400; border:2px solid red; background-color: #f8f8f8;">{}</font><br><br>' .format(message)) break loglines.append(line.strip()) count += 1 self._append_to_process_log(test_pid, test_ppid, loglines) self._state[testname]['last_log_line'] = count if status in self.result_keys: links = [] for link in test_links: html_link = '[OPERETO_HTML]<br><a href="{}"><font style="color: #222; font-weight: 600; font-size: 13px;">{}</font></a>'.format( link['url'], link['name']) links.append(html_link) self._append_to_process_log(test_pid, test_ppid, links) self.client.stop_process(test_pid, status=status) self._state[testname]['status'] = status def process(self): def process_results(): try: tests_json = os.path.join(self.test_results_dir, 'tests.json') if os.path.exists(tests_json): with open(tests_json, 'r') as tf: try: self.test_data = json.load(tf) self.op_state['test_data'] = self.test_data try: validator = JsonSchemeValidator( self.test_data, self.tests_json_scheme) validator.validate() except Exception as e: print('Invalid tests json file: {}'.format(e)) return if 'test_records' in self.test_data: for test_record in self.test_data[ 'test_records']: self._modify_record(test_record) if 'test_suite' in self.test_data: if 'status' in self.test_data['test_suite']: self.op_state[ 'test_suite_final_status'] = self.test_data[ 'test_suite']['status'] if 'links' in self.test_data['test_suite']: self.suite_links = self.test_data[ 'test_suite']['links'] self.op_state[ 'test_suite'] = self.suite_links finally: try: self._save_state(self.op_state) except Exception as e: print(e) finally: self.end_of_test_suite = self.client.get_process_property( name='stop_listener_code') if self.debug_mode: print('[DEBUG] content of tests.json: {}'.format( json.dumps(self.test_data))) while (True): process_results() time.sleep(self.client.input['listener_frequency']) if self.end_of_test_suite: print('Stopping listener process..') break def setup(self): self._print_step_title('Start opereto test listener..') if not os.path.exists(self.input['test_results_path']): make_directory(self.input['test_results_path']) self.op_state = { 'test_suite_final_status': 'success', 'test_results_path': self.input['test_results_path'], 'test_data': {}, 'suite_links': [] } self._save_state(self.op_state) def teardown(self): if 'test_results_path' in self.op_state: remove_directory_if_exists(self.input['test_results_path']) self._print_step_title('Opereto test listener stopped.') print('Final content of tests_json: {}'.format(json.dumps( self.op_state['test_data']), indent=4)) suite_links = self.op_state.get('suite_links') or [] for link in suite_links: self._print_test_link(link) print('Final listener status is {}'.format( self.op_state['test_suite_final_status'])) if self.op_state['test_suite_final_status'] == 'success': return self.client.SUCCESS elif self.op_state['test_suite_final_status'] == 'failure': return self.client.FAILURE elif self.op_state['test_suite_final_status'] == 'warning': return self.client.WARNING else: return self.client.ERROR
def __init__(self): self.client = OperetoClient() self.input = self.client.input
class MyFlowService(): def __init__(self): self.client = OperetoClient() self.input = self.client.input self.failed = False def run(self): try: def _end_fake_parent_process(parent_pid, child_process_pids=[]): if self.client.is_success(child_process_pids): self.client.stop_process(parent_pid, 'success') else: self.failed = True self.client.stop_process(parent_pid, 'failure') ## create a "fake" parent process parent_1_pid = self.client.create_process( 'fake_parent_process', title='First commands group') ## attach some services to it as child processes cmd_1_pid = self.client.create_process('shell_command', title='df -k', command='df -k', pflow_id=parent_1_pid) cmd_2_pid = self.client.create_process('shell_command', title='netstat -rn', command='netstat -rn', pflow_id=parent_1_pid) _end_fake_parent_process(parent_1_pid, [cmd_1_pid, cmd_2_pid]) time.sleep(10) ## create a second "fake" parent process parent_2_pid = self.client.create_process( 'fake_parent_process', title='Second commands group') ## attach some services to it as child processes cmd_3_pid = self.client.create_process('shell_command', title='df -k', command='df -k', pflow_id=parent_2_pid) cmd_4_pid = self.client.create_process('shell_command', title='netstat -rn', command='netstat -rn', pflow_id=parent_2_pid) _end_fake_parent_process(parent_2_pid, [cmd_3_pid, cmd_4_pid]) time.sleep(5) ## set final result if self.failed: return self.client.FAILURE return self.client.SUCCESS except: traceback.format_exc() return self.client.FAILURE
usage = "%prog -s service [-t title] [-a agent] [-v version] [--production]" parser = OptionParser(usage=usage) parser.add_option("-s", "--service", dest="service", help="service name.") parser.add_option("-v", "--version", dest="version", help="service version specification (optional), for instance: 1.1.0-12.") parser.add_option("-t", "--title", dest="title", help="process title (optional).") parser.add_option("-a", "--agent", dest="agent", default='any', help="agent name (optional), default is 'any'.") parser.add_option("--production", dest="production", action='store_true', default=False, help="Runs the default production version. The same as specifying [--version default]") (options, args) = parser.parse_args() if not options.service: parser.error('service must be provided.') return (options, args) if __name__ == "__main__": (options, args) = parse_options() client = OperetoClient() operations_mode = 'development' if options.production or options.version: operations_mode = 'production' version=options.version if not version: version='default' pid = client.create_process(service=options.service, service_version=version, title=options.title, agent=options.agent , mode=operations_mode) if operations_mode=='production': print 'A new process for service [%s] has been created: mode=%s, version=%s, pid=%s'%(options.service, operations_mode, options.version, pid) else: print 'A new development process for service [%s] has been created: pid=%s'%(options.service, pid)
def parse_options(): usage = "%prog -d path/to/service/directory [-s service] [-v version] [--production]" parser = OptionParser(usage=usage) parser.add_option("-d", "--dir", dest="dir", help="path to the service directory.") parser.add_option("-s", "--service", dest="service", help="service name (optional), if not provided, directory name will be used.") parser.add_option("-v", "--version", dest="version", help="service version specification, for instance: 1.1.0-12.") parser.add_option("--production", dest="production", action='store_true', default=False, help="Uploads default version to production (the same as specifying: --version default)") (options, args) = parser.parse_args() if not options.dir: parser.error('service directory must be provided.') return (options, args) if __name__ == "__main__": (options, args) = parse_options() client = OperetoClient() zip_action_file = os.path.join(TEMP_DIR, str(uuid.uuid4())+'.action') zipfolder(zip_action_file, options.dir) operations_mode = 'development' if options.production or options.version: operations_mode = 'production' version=options.version if not version: version='default' client.upload_service_version(service_zip_file=zip_action_file+'.zip', mode=operations_mode, service_version=version, service_id=options.service) if operations_mode=='production': print 'Service production version [%s] deployed successfuly.'%version else: print 'Service development version deployed successfully.'
def __init__(self, **kwargs): self.client = OperetoClient() ServiceTemplate.__init__(self, **kwargs)
def opereto_client(): return OperetoClient()