class MyService():
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
print self.input
def run(self):
try:
## print the input parameter
print 'The input parameters passed: %s' % str(
self.input['my_input_param'])
## modify output parameter with input one
self.client.modify_process_property('my_output_param',
self.input['my_input_param'])
## verify that input text started with Hello World
if not self.input['my_input_param'].startswith('Hello World'):
print >> sys.stderr, 'Input does not start with Hello World'
return self.client.FAILURE
return self.client.SUCCESS
except:
traceback.format_exc()
return self.client.FAILURE
def get_opereto_client():
if opereto_credentials_json.get('opereto_password'):
client = OperetoClient(opereto_host=OPERETO_HOST,
opereto_user=opereto_credentials_json['opereto_user'],
opereto_password=opereto_credentials_json['opereto_password'])
return client
else:
client = OperetoClient(opereto_host=OPERETO_HOST, opereto_token=opereto_credentials_json['opereto_token'])
return client
class ProcessLogScheduler():
"""
from time import sleep
with ProcessLogScheduler(1) as p:
p.write('eAsEFPGfJun', [{'text': 'my log entry 1', 'level': 'INFO'}])
p.write('eAsEFPGfJun', [{'text': 'my log entry 2', 'level': 'INFO'}])
p.write('53cqHRdu4Cr', [{'text': 'some log text..', 'level': 'ERROR'}])
sleep(5) # your long-running job goes here...
"""
def __init__(self, interval=5):
self.buffer = {
0: {},
1: {}
}
self.active_buffer = 0
self._timer = None
self.client = OperetoClient()
self.is_running = False
self.interval = interval
self.start()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.flush_logs()
def _switch_buffer(self):
active_buffer_sn = self.active_buffer
self.active_buffer = abs(int(self.active_buffer) - 1)
logs = self.buffer[active_buffer_sn]
self.buffer[active_buffer_sn]={}
return logs
def flush_logs(self, *args):
self.is_running = False
self.start()
for pid, entries in self._switch_buffer().items():
self.client.send_process_log(pid, entries)
def start(self):
if not self.is_running:
self._timer = Timer(self.interval, self.flush_logs)
self._timer.daemon = True
self._timer.start()
self.is_running = True
def stop(self):
self._timer.cancel()
self.is_running = False
def write(self, pid, log_entries=[]):
if not pid in self.buffer[self.active_buffer]:
self.buffer[self.active_buffer][pid]=[]
self.buffer[self.active_buffer][pid]+=log_entries
def remove_opereto_lib():
try:
c = OperetoClient()
c.modify_agent_property(c.input['opereto_agent'], 'opereto.worker',
False)
return c.SUCCESS
except Exception as e:
print(e)
return 2
def __init__(self, interval=5):
self.buffer = {
0: {},
1: {}
}
self.active_buffer = 0
self._timer = None
self.client = OperetoClient()
self.is_running = False
self.interval = interval
self.start()
def get_opereto_client():
try:
OPERETO_AUTH_TOKEN = opereto_credentials_json['opereto_auth_token']
client = OperetoClient(opereto_host=OPERETO_HOST,
opereto_auth_token=OPERETO_AUTH_TOKEN)
except KeyError:
opereto_user = opereto_credentials_json['opereto_user']
opereto_password = opereto_credentials_json['opereto_password']
client = OperetoClient(opereto_host=OPERETO_HOST,
opereto_user=opereto_user,
opereto_password=opereto_password)
return client
def __init__(self, **kwargs):
self.client = OperetoClient()
self.input = self.client.input
self.exitcode = 0
if kwargs:
self.input.update(kwargs)
def __init__(self, **kwargs):
self.client = OperetoClient()
self.input = self.client.input
self.state_file = os.path.join(self.input['opereto_workspace'],
'_opereto_service_state.json')
self.exitcode = 0
if kwargs:
self.input.update(kwargs)
class MyService:
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
def run(self):
try:
## print the input parameter
print "The input parameters passed: %s" % str(self.input["my_input_param"])
## modify output parameter with input one
self.client.modify_process_property("my_output_param", self.input["my_input_param"])
## verify that input text started with Hello World
if not self.input["my_input_param"].startswith("Hello World"):
print >> sys.stderr, "Input does not start with Hello World"
return self.client.FAILURE
return self.client.SUCCESS
except:
traceback.format_exc()
return self.client.FAILURE
import os
from pyopereto.client import OperetoClient
from pyopereto.helpers.packages import OperetoAwsS3PackagesManager
package_directory = os.path.dirname(os.getcwd())
## set the AWS S3 bucket name, access key and secret key
client = OperetoClient()
opereto_packages_repo_ak = client.input['opereto_packages_repo_ak']
opereto_packages_repo_sk = client.input['opereto_packages_repo_sk']
opereto_packages_bucket_name = client.input['opereto_packages_bucket_name']
if __name__ == "__main__":
obj = OperetoAwsS3PackagesManager(package_directory,
opereto_packages_bucket_name,
opereto_packages_repo_ak,
opereto_packages_repo_sk,
'opereto-gcp-services')
obj.deploy()
def install_opereto_lib():
LIB_DIR = os.path.join(os.environ['opereto_workspace'], 'opereto')
VIRT_ENV_DIR = os.path.join(os.environ['opereto_home'], 'operetovenv')
per_os_module_to_install = {
'rhel7.2': {
'paramiko': 'paramiko==2.1.2'
}
}
module_to_install = {
'yaml' : 'pyyaml==3.13',
'pyopereto': 'pyopereto==1.0.55',
'requests': 'requests==2.19.1',
'boto': 'boto==2.49.0',
'paramiko': 'paramiko==2.4.1',
'six': 'six==1.11.0',
'sh': 'sh==1.12.14',
'werkzeug': 'werkzeug==0.14.1',
'psutil': 'psutil==0.7.0',
'dateutil': 'python-dateutil==2.7.3',
'jsonschema': 'jsonschema==2.6.0',
'faker_schema': 'faker-schema==0.1.4',
'docker': 'docker==3.1.4',
'PIL': 'Pillow==2.9.0',
'boto3': 'boto3==1.7.39',
'httplib2': 'httplib2==0.11.3',
'oauth2client': 'oauth2client==4.1.2',
'apiclient': 'google-api-python-client==1.7.4',
'junitparser': 'junitparser==1.2.2',
'kubernetes':'kubernetes==5.0.0'
}
def get_current_os():
(name, version,id) = platform.linux_distribution()
return (name, version,id)
def is_ubuntu():
(name, version,id) = get_current_os()
if name == 'Ubuntu':
return True
return False
def is_windows():
if platform.system() == 'Windows':
return True
return False
def _remove_dir(dir):
if os.path.exists(dir):
if is_windows():
_local("rmdir /s /q %s"%dir)
else:
_local("sudo rm -rf %s"%dir)
def _create_dir_if_not_exists(dir):
if not os.path.exists(dir):
if is_windows():
_local("mkdir %s" % dir)
else:
_local("sudo mkdir -p %s" % dir)
_local('sudo chmod -R 777 ' + dir)
def _copy_opereto_venv_contents(dir):
if is_windows():
WIN_LIB_DIR = LIB_DIR.replace("/","\\")
_local("xcopy %s %s /q /i /e /Y" % (WIN_LIB_DIR, os.path.join(dir, 'opereto', 'venv')))
else:
_local("cp -rf %s %s" % (LIB_DIR, os.path.join(dir, 'opereto')))
def _local(cmd, ignore=False):
global FAILURE
print cmd
ret = os.system(cmd)
if int(ret)!=0 and not ignore:
print 'Command ended with exit code: {}'.format(int(ret))
FAILURE=True
return ret
def _prepare_new_virtual_env(directory):
os_name=None
(name, version,id) = get_current_os()
print 'Current OS: {} {}, {}'.format(name, version,id)
if name=='Red Hat Enterprise Linux Server' and version=='7.2':
os_name='rhel7.2'
if os_name:
module_to_install.update(per_os_module_to_install[os_name])
if is_windows():
_local('cd %s && virtualenv venv' % (directory), ignore=False)
for import_name, module in module_to_install.items():
_local('cd %s && pip install %s' % (os.path.join(directory,'venv', 'Scripts'), module), ignore=False)
if _local('cd %s && python -c "import %s"' % (os.path.join(directory, 'venv', 'Scripts'), import_name)) != 0:
print >> sys.stderr, 'Python module [%s] is not installed.' % module
else:
_local('cd %s && virtualenv venv && . venv/bin/activate && pip install --upgrade pip ; deactivate'%(directory),ignore=False)
for import_name, module in module_to_install.items():
_local('cd %s && . venv/bin/activate && pip install %s && deactivate'%(directory, module),ignore=True)
if _local('cd %s && . venv/bin/activate && python -c "import %s" && deactivate'%(directory, import_name))!=0:
print >> sys.stderr, 'Python module [%s] is not installed.'%module
def create_virtual_rlauto_env(current_version_dir):
_remove_dir(current_version_dir)
_create_dir_if_not_exists(current_version_dir)
_copy_opereto_venv_contents(current_version_dir)
_prepare_new_virtual_env(current_version_dir)
if not is_windows():
_local('sudo chmod 777 -R %s'%current_version_dir)
_local("sudo su -c 'echo \"export PYTHONPATH=%s\" >> %s'"%(current_version_dir,os.path.join(current_version_dir,'venv/bin/activate')))
try:
release=os.environ.get('opereto_service_version')
## check prerequisites
if sys.version_info<(2,7):
print >> sys.stderr,'Opereto microservices lib requires python 2.7 or higher.'
return 2
if is_ubuntu():
(name, version,id) = get_current_os()
install_list = 'sudo apt-get install -qy python-six curl python-setuptools gcc build-essential python-dev python-pip libffi-dev libssl-dev'
_local(install_list,ignore=False)
_local('sudo pip install virtualenv', ignore=False)
elif is_windows():
_local('pip install virtualenv', ignore=False)
_local('pip install pyopereto')
else:
_local('sudo yum install -y python-virtualenv python-setuptools gcc libffi python-devel openssl-devel')
_local('easy_install pip')
if not is_windows():
_local('sudo pip install pyopereto')
current_version_dir= os.path.join(VIRT_ENV_DIR,release)
create_virtual_rlauto_env(current_version_dir)
if FAILURE:
return 3
else:
if os.environ.get('opereto_agent'):
from pyopereto.client import OperetoClient
c = OperetoClient()
if c.input['standard_opereto_worker']:
c.modify_agent_property(c.input['opereto_agent'], 'opereto.worker', True)
return 0
except Exception, e:
print traceback.format_exc()
return 2
class ServiceRunner(ServiceTemplate):
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"pod_operation": {
"enum": ['create_pod', 'delete_pod']
},
"pod_name": {
"type": ["null", "string"]
},
"pod_template": {
"type": "object"
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'pod_operation', 'pod_template', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['pod_operation'] == 'delete_pod' and not self.input.get(
'pod_name'):
raise OperetoRuntimeError(
error='Pod name must be provided for this operation')
if self.input['pod_name'].startswith('opereto-worker-node'):
raise OperetoRuntimeError(
error=
'Pod name is invalid, this name is used for Opereto standard elastic workers. Please select a different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def process(self):
def _modify_agent(agent_id):
self.client.create_agent(
agent_id=agent_id,
name=agent_id,
description=
'This agent worker is part of {} worker stateful set.'.format(
self.pod_name))
time.sleep(2)
agent_properties = self.input['agent_properties']
agent_properties.update({
'opereto.shared': True,
'worker.label': self.pod_name
})
self.client.modify_agent_properties(agent_id, agent_properties)
def _agents_status(online=True):
while (True):
try:
agent_attr = self.client.get_agent(self.pod_name)
if agent_attr['online'] == online:
break
except OperetoClientError:
pass
time.sleep(5)
def _tearrdown_pod():
print 'Deleting worker pod..'
self.pod_info = self.kubernetes_api.delete_pod(self.pod_name)
print 'Waiting that worker pod will be offline (may take some time)..'
_agents_status(online=False)
print 'Agent {} is offline.'.format(self.pod_name)
if self.pod_operation == 'create_pod':
print 'Creating worker pod..'
_modify_agent(self.pod_name)
self.kubernetes_api.create_pod(self.pod_template)
print 'Waiting that worker pod will be online (may take some time)..'
_agents_status(online=True)
print 'Agent {} is online.'.format(self.pod_name)
self.pod_info = self.kubernetes_api.get_pod(self.pod_name)
print self.pod_info.status
## run post install services
for service in self.input['post_operations']:
input = service.get('input') or {}
agent = service.get('agents') or self.pod_name
pid = self.client.create_process(service=service['service'],
agent=agent,
title=service.get('title'),
**input)
if not self.client.is_success(pid):
_tearrdown_pod()
return self.client.FAILURE
elif self.pod_operation == 'delete_pod':
_tearrdown_pod()
else:
raise OperetoRuntimeError(
error='Invalid operation: {}'.format(self.pod_operation))
return self.client.SUCCESS
def setup(self):
self.kubernetes_api = KubernetesAPI()
self.pod_name = self.input['pod_name']
self.pod_operation = self.input['pod_operation']
self.pod_info = {}
self.pod_template = self.input['pod_template']
if self.pod_operation == 'create_pod':
if self.pod_name:
self.pod_template["metadata"]["name"] = self.pod_name
self.pod_template["spec"]["containers"][0][
"name"] = self.pod_name + "-worker"
else:
self.pod_name = self.pod_template["metadata"]["name"]
if not self.pod_template["spec"]["containers"][0].get('env'):
self.pod_template["spec"]["containers"][0]['env'] = []
self.pod_template["spec"]["containers"][0]['env'] += [{
"name": "agent_name",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}, {
"name":
"opereto_host",
"value":
self.input['opereto_host']
}, {
"name": "opereto_user",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_USERNAME"
}
}
}, {
"name": "opereto_password",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_PASSWORD"
}
}
}, {
"name":
"javaParams",
"value":
self.input['agent_java_config'],
}, {
"name":
"log_level",
"value":
self.input['agent_log_level']
}]
print 'Pod template:\n{}'.format(
json.dumps(self.pod_template, indent=4))
def teardown(self):
print self.pod_info
#!/usr/bin/python
from pyopereto.client import OperetoClient
from optparse import OptionParser
def parse_options():
usage = "%prog -s service [-v version]"
parser = OptionParser(usage=usage)
parser.add_option("-s", "--service", dest="service", help="service name.")
parser.add_option("-v", "--version", dest="version", help="delete a specific service production version, for instance: 1.1.0-12.")
(options, args) = parser.parse_args()
if not options.service:
parser.error('service must be provided.')
return (options, args)
if __name__ == "__main__":
(options, args) = parse_options()
client = OperetoClient()
operations_mode = 'development'
if options.version:
operations_mode = 'production'
version=options.version
if not version:
version='default'
client.delete_service_version(service_id=options.service, service_version=version, mode=operations_mode)
if operations_mode=='production':
print 'Version [%s] of service [%s] has been deleted'%(options.service, options.version)
else:
print 'Development version of service [%s] has been deleted'%options.service
def get_opereto_client():
client = OperetoClient(
opereto_host=opereto_credentials_json['opereto_host'],
opereto_user=opereto_credentials_json['opereto_user'],
opereto_password=opereto_credentials_json['opereto_password'])
return client
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
self.sflow_id = self.input['opereto_source_flow_id']
self.remove_test_results_dir = False
self.op_state = self._get_state() or {}
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
self.failed = False
def install_opereto_lib():
LIB_DIR = os.path.join(os.environ['opereto_workspace'], 'opereto')
VIRT_ENV_DIR = os.path.join(os.environ['opereto_home'], 'operetovenv')
module_to_install = {
'boto3': 'boto3==1.20.6',
'yaml': 'pyyaml==6.0',
'pyopereto': 'pyopereto==1.0.132',
'kubernetes': 'kubernetes==19.15.0',
'requests': 'requests==2.26.0',
'paramiko': 'paramiko==2.8.0',
'six': 'six==1.16.0',
'sh': 'sh==1.14.2',
'werkzeug': 'werkzeug==2.0.2',
'psutil': 'psutil==5.8.0',
'dateutil': 'python-dateutil==2.8.2',
'jsonschema': 'jsonschema==2.6.0',
'faker_schema': 'faker-schema==0.1.4',
'docker': 'docker==5.0.3',
'PIL': 'Pillow==8.4.0',
'httplib2': 'httplib2==0.20.2',
'oauth2client': 'oauth2client==4.1.3',
'apiclient': 'google-api-python-client==2.30.0',
'junitparser': 'junitparser==2.1.1'
}
def get_pip_command():
if version == '20.04':
return 'pip3'
return 'pip3.8'
def get_virtenv_cmd():
return 'virtualenv'
def get_current_os():
out = subprocess.check_output('cat /etc/*-release',
shell=True).decode('utf-8')
name = re.search('NAME="([A-Za-z0-9\s]*)"', out)
version = re.search('VERSION_ID=\"([\d\.\-]*)\"', out)
os_id = re.search('ID=([A-Za-z0-9\s\"]*)\n', out)
if name and version and os_id:
return name.group(1), version.group(1), os_id.group(1).strip('"')
else:
raise Exception(
f'cant find OS details (NAME, VERSION_ID, ID), check full os-release output:\n{out}'
)
def _remove_dir(dir):
if os.path.exists(dir):
_local("sudo rm -rf {}".format(dir))
def _create_dir_if_not_exists(dir):
if not os.path.exists(dir):
_local("sudo mkdir -p {}".format(dir))
_local('sudo chmod -R 777 ' + dir)
def _copy_opereto_venv_contents(dir):
_local("cp -rf {} {}".format(LIB_DIR, os.path.join(dir, 'opereto')))
def _local(cmd, ignore=False):
global FAILURE
print(cmd)
ret = os.system(cmd)
if int(ret) != 0 and not ignore:
print('Command ended with exit code: {}'.format(int(ret)))
FAILURE = True
return ret
def _prepare_new_virtual_env(directory):
(name, version, id) = get_current_os()
print('Current OS: {} {}, {}'.format(name, version, id))
_local(
'cd {} && {} venv -p python3.6 && . venv/bin/activate && pip install --upgrade pip ; deactivate'
.format(directory, get_virtenv_cmd()),
ignore=False)
for import_name, module in module_to_install.items():
_local(
'cd {} && . venv/bin/activate && pip install {} && deactivate'.
format(directory, module),
ignore=True)
if _local(
'cd {} && . venv/bin/activate && python -c "import {}" && deactivate'
.format(directory, import_name)) != 0:
sys.stderr.write(
'Python module [{}] is not installed.'.format(module))
def create_virtual_env(current_version_dir):
_remove_dir(current_version_dir)
_create_dir_if_not_exists(current_version_dir)
_copy_opereto_venv_contents(current_version_dir)
_prepare_new_virtual_env(current_version_dir)
_local('sudo chmod 777 -R {}'.format(current_version_dir))
_local("sudo su -c 'echo \"export PYTHONPATH={}\" >> {}'".format(
current_version_dir,
os.path.join(current_version_dir, 'venv/bin/activate')))
try:
release = os.environ.get('opereto_service_version')
(name, version, id) = get_current_os()
if sys.version_info < (3, 6):
sys.stderr.write(
'Opereto microservices lib requires python 3.6 or higher.')
return 2
_local(
'export DEBIAN_FRONTEND=noninteractive ; sudo -E apt-get update')
install_list = 'sudo apt-get install -qy python-six curl python-setuptools gcc build-essential python3.6-dev libffi-dev libssl-dev'
_local(install_list, ignore=False)
_local('curl https://bootstrap.pypa.io/get-pip.py | sudo -H python3.8')
_local(
'export LC_ALL=\"en_US.UTF-8\" ; export LC_CTYPE=\"en_US.UTF-8\" ; sudo {} install virtualenv==16.0.0'
.format(get_pip_command()),
ignore=False)
current_version_dir = os.path.join(VIRT_ENV_DIR, release)
create_virtual_env(current_version_dir)
if FAILURE:
return 3
try:
_local('{} install --upgrade pyopereto==1.0.124'.format(
get_pip_command()))
if os.environ.get('opereto_agent'):
from pyopereto.client import OperetoClient
c = OperetoClient()
if c.input['standard_opereto_worker']:
c.modify_agent_property(c.input['opereto_agent'],
'opereto.worker', True)
except Exception as e:
print(e)
return 0
except Exception as e:
print(traceback.format_exc())
return 2
class ServiceRunner(ServiceTemplate):
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"deployment_operation": {
"enum": [
'create_statefulset', 'modify_statefulset',
'delete_statefulset', 'update_worker_image'
]
},
"deployment_name": {
"type": ["null", "string"]
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'deployment_operation', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['deployment_name'] == 'opereto-worker-node':
raise OperetoRuntimeError(
error=
'Deployment name is invalid, this name is used for Opereto standard workers. Please insert different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def process(self):
def _get_agent_names():
names = []
for count in range(self.worker_replicas):
names.append(self.deployment_name + '-' + str(count))
return names
def _modify_agent(agent_id):
try:
self.client.get_agent(agent_id)
except OperetoClientError:
self.client.create_agent(
agent_id=agent_id,
name=agent_id,
description=
'This agent worker is part of {} worker stateful set.'.
format(self.deployment_name))
time.sleep(2)
agent_properties = self.input['agent_properties']
agent_properties.update({
'opereto.shared': True,
'worker.label': self.deployment_name
})
self.client.modify_agent_properties(agent_id, agent_properties)
def _agents_status(online=True):
while (True):
ok = True
for agent_id in _get_agent_names():
try:
agent_attr = self.client.get_agent(agent_id)
if agent_attr['online'] != online:
ok = False
break
except OperetoClientError:
pass
if ok:
break
time.sleep(5)
def _tearrdown_statefileset():
print 'Deleting worker stateful set..'
self.deployment_info = self.kubernetes_api.delete_stateful_set(
self.deployment_name)
print 'Waiting that all worker pods will be offline (may take some time)..'
_agents_status(online=False)
if self.deployment_operation == 'create_statefulset':
print 'Creating worker stateful set..'
self.deployment_info = self.kubernetes_api.create_stateful_set(
self.deployment_template)
for agent_id in _get_agent_names():
_modify_agent(agent_id)
print 'Waiting that all worker pods will be online (may take some time)..'
_agents_status(online=True)
self.deployment_info = self.kubernetes_api.get_stateful_set(
self.deployment_name)
print self.deployment_info.status
## run post install services
post_install_pids = []
for agent_id in _get_agent_names():
for service in self.input['post_operations']:
input = service.get('input') or {}
agent_name = service.get('agents') or agent_id
pid = self.client.create_process(
service=service['service'],
agent=agent_name,
title=service.get('title'),
**input)
post_install_pids.append(pid)
if post_install_pids and not self.client.is_success(
post_install_pids):
_tearrdown_statefileset()
return self.client.FAILURE
elif self.deployment_operation == 'modify_statefulset':
print 'Modifying worker stateful set..'
self.deployment_info = self.kubernetes_api.modify_stateful_set(
self.deployment_name, self.deployment_template)
for agent_id in _get_agent_names():
_modify_agent(agent_id)
print 'Waiting that all worker pods will be online (may take some time)..'
_agents_status(online=True)
self.deployment_info = self.kubernetes_api.get_stateful_set(
self.deployment_name)
print self.deployment_info.status
elif self.deployment_operation == 'delete_statefulset':
_tearrdown_statefileset()
else:
raise OperetoRuntimeError(error='Invalid operation: {}'.format(
self.deployment_operation))
return self.client.SUCCESS
def setup(self):
self.kubernetes_api = KubernetesAPI()
self.deployment_name = self.input['deployment_name']
self.deployment_operation = self.input['deployment_operation']
self.deployment_info = {}
self.deployment_template = self.input['deployment_template']
self.worker_replicas = self.deployment_template["spec"]["replicas"]
if self.deployment_operation in [
'create_statefulset', 'modify_statefulset'
]:
if self.deployment_name:
self.deployment_template["metadata"][
"name"] = self.deployment_name
self.deployment_template["spec"]["template"]["spec"][
"containers"][0]["name"] = self.deployment_name + "-worker"
try:
self.deployment_template["spec"]["selector"][
"matchLabels"][
"app"] = self.deployment_name + "-cluster"
except:
pass
try:
self.deployment_template["spec"]["template"]['metadata'][
"labels"]["app"] = self.deployment_name + "-cluster"
except:
pass
else:
self.deployment_name = self.deployment_template["metadata"][
"name"]
if not self.deployment_template["spec"]["template"]["spec"][
"containers"][0].get('env'):
self.deployment_template["spec"]["template"]["spec"][
"containers"][0]['env'] = []
self.deployment_template["spec"]["template"]["spec"]["containers"][
0]['env'] += [{
"name": "agent_name",
"valueFrom": {
"fieldRef": {
"fieldPath": "metadata.name"
}
}
}, {
"name": "opereto_host",
"value": self.input['opereto_host']
}, {
"name": "opereto_user",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_USERNAME"
}
}
}, {
"name": "opereto_password",
"valueFrom": {
"secretKeyRef": {
"name": self.input['worker_config'],
"key": "OPERETO_PASSWORD"
}
}
}, {
"name": "javaParams",
"value": self.input['agent_java_config'],
}, {
"name": "log_level",
"value": self.input['agent_log_level']
}]
print 'Deployment template:\n{}'.format(
json.dumps(self.deployment_template, indent=4))
def teardown(self):
print self.deployment_info
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
print self.input
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
self._print_step_title('Start opereto test listener..')
self.sflow_id = self.input['opereto_source_flow_id']
self.remove_test_results_dir=False
class ServiceRunner(ServiceTemplate):
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
self.sflow_id = self.input['opereto_source_flow_id']
self.remove_test_results_dir = False
self.op_state = self._get_state() or {}
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"test_results_path": {
"type": "string",
"minLength": 1
},
"parent_pid": {
"type": "string"
},
"listener_frequency": {
"type": "integer",
"minValue": 1
},
"debug_mode": {
"type": "boolean"
}
},
"required": ['listener_frequency', 'test_results_path'],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.parent_pid = self.input['parent_pid'] or self.input['pid']
self.test_results_dir = self.input['test_results_path']
self.debug_mode = self.input['debug_mode']
self.result_keys = process_result_keys
self.status_keys = process_status_keys
self.tests_json_scheme = {
"type": "object",
"properties": {
"test_suite": {
"type": "object",
"properties": {
"links": {
"type": "array"
},
"status": {
"enum": self.status_keys
}
}
},
"test_records": {
"type":
"array",
"items": [{
"type": "object",
"properties": {
"testname": default_entity_name_scheme,
"status": {
"enum": self.status_keys
},
"title": default_entity_name_scheme,
"links": {
"type":
"array",
"items": [{
"type": "object",
"properties": {
"url": {
"type": "string"
},
"name": {
"type": "string"
}
}
}]
}
},
"required": ['testname', 'status'],
"additionalProperties": True
}]
}
},
"additionalProperties": True
}
self.end_of_test_suite = None
self.test_data = {}
self.suite_links = []
self._state = {}
def _print_test_link(self, link):
print(
'[OPERETO_HTML]<br><a href="{}"><font style="color: #222; font-weight: 600; font-size: 13px;">{}</font></a>'
.format(link['url'], link['name']))
def _append_to_process_log(self, pid, ppid, loglines, log_level='info'):
log_request_data = {
'sflow_id': self.sflow_id,
'pflow_id': ppid,
'agent_id': self.input['opereto_agent'],
'product_id': self.input['opereto_product_id'],
'data': []
}
count = 1
for line in loglines:
try:
millis = int(round(time.time() * 1000)) + count
log_request_data['data'].append({
'level': log_level,
'text': line.strip(),
'timestamp': millis
})
except Exception as e:
print(e)
count += 1
self.client._call_rest_api(
'post',
'/processes/{}/log'.format(pid),
data=log_request_data,
error='Failed to update test log (test pid = {})'.format(pid))
def _modify_record(self, test_record):
testname = test_record['testname']
test_input = test_record.get('test_input') or {}
title = test_record.get('title') or testname
status = test_record['status']
test_links = test_record.get('links') or []
test_ppid = test_record.get('ppid') or self.parent_pid
test_pid = test_record.get('pid')
if testname not in self._state:
if not test_pid:
test_pid = self.client.create_process(
'opereto_test_listener_record',
testname=testname,
title=title,
test_input=test_input,
test_runner_id=test_ppid,
pflow_id=test_ppid)
self.client.wait_to_start(test_pid)
self._state[testname] = {
'ppid': test_ppid,
'pid': test_pid,
'status': 'in_process',
'title': title,
'test_output_md5': '',
'summary_md5': '',
'last_log_line': 1
}
else:
test_pid = self._state[testname]['pid']
if self._state[testname]['status'] not in self.result_keys:
if title != self._state[testname]['title']:
### TBD: add title change API call
self._state[testname]['title'] = title
results_dir = os.path.join(self.test_results_dir, testname)
if os.path.exists(results_dir):
output_json_file = os.path.join(results_dir, 'output.json')
log_file = os.path.join(results_dir, 'stdout.log')
summary_file = os.path.join(results_dir, 'summary.txt')
if os.path.exists(output_json_file):
output_json_md5 = get_file_md5sum(output_json_file)
if output_json_md5 != self._state[testname][
'test_output_md5']:
with open(output_json_file, 'r') as of:
output_json = json.load(of)
self.client.modify_process_property('test_output',
output_json,
pid=test_pid)
self._state[testname][
'test_output_md5'] = output_json_md5
if os.path.exists(summary_file):
summary_md5 = get_file_md5sum(summary_file)
if summary_md5 != self._state[testname]['summary_md5']:
with open(summary_file, 'r') as sf:
summary = sf.read()
self.client.modify_process_summary(
test_pid, summary)
self._state[testname]['summary_md5'] = summary_md5
if os.path.exists(log_file):
with open(log_file, 'r') as lf:
count = 1
loglines = []
for line in lf.readlines():
if count >= self._state[testname]['last_log_line']:
if count > MAX_LOG_LINES_PER_PROCESS:
message = 'Test log is too long. Please save test log in remote storage and add a link to it in Opereto log. See service info to learn how to add links to your tests.json file.'
loglines.append(
'[OPERETO_HTML]<br><br><font style="width: 800px; padding: 15px; color: #222; font-weight: 400; border:2px solid red; background-color: #f8f8f8;">{}</font><br><br>'
.format(message))
break
loglines.append(line.strip())
count += 1
self._append_to_process_log(test_pid, test_ppid,
loglines)
self._state[testname]['last_log_line'] = count
if status in self.result_keys:
links = []
for link in test_links:
html_link = '[OPERETO_HTML]<br><a href="{}"><font style="color: #222; font-weight: 600; font-size: 13px;">{}</font></a>'.format(
link['url'], link['name'])
links.append(html_link)
self._append_to_process_log(test_pid, test_ppid, links)
self.client.stop_process(test_pid, status=status)
self._state[testname]['status'] = status
def process(self):
def process_results():
try:
tests_json = os.path.join(self.test_results_dir, 'tests.json')
if os.path.exists(tests_json):
with open(tests_json, 'r') as tf:
try:
self.test_data = json.load(tf)
self.op_state['test_data'] = self.test_data
try:
validator = JsonSchemeValidator(
self.test_data, self.tests_json_scheme)
validator.validate()
except Exception as e:
print('Invalid tests json file: {}'.format(e))
return
if 'test_records' in self.test_data:
for test_record in self.test_data[
'test_records']:
self._modify_record(test_record)
if 'test_suite' in self.test_data:
if 'status' in self.test_data['test_suite']:
self.op_state[
'test_suite_final_status'] = self.test_data[
'test_suite']['status']
if 'links' in self.test_data['test_suite']:
self.suite_links = self.test_data[
'test_suite']['links']
self.op_state[
'test_suite'] = self.suite_links
finally:
try:
self._save_state(self.op_state)
except Exception as e:
print(e)
finally:
self.end_of_test_suite = self.client.get_process_property(
name='stop_listener_code')
if self.debug_mode:
print('[DEBUG] content of tests.json: {}'.format(
json.dumps(self.test_data)))
while (True):
process_results()
time.sleep(self.client.input['listener_frequency'])
if self.end_of_test_suite:
print('Stopping listener process..')
break
def setup(self):
self._print_step_title('Start opereto test listener..')
if not os.path.exists(self.input['test_results_path']):
make_directory(self.input['test_results_path'])
self.op_state = {
'test_suite_final_status': 'success',
'test_results_path': self.input['test_results_path'],
'test_data': {},
'suite_links': []
}
self._save_state(self.op_state)
def teardown(self):
if 'test_results_path' in self.op_state:
remove_directory_if_exists(self.input['test_results_path'])
self._print_step_title('Opereto test listener stopped.')
print('Final content of tests_json: {}'.format(json.dumps(
self.op_state['test_data']),
indent=4))
suite_links = self.op_state.get('suite_links') or []
for link in suite_links:
self._print_test_link(link)
print('Final listener status is {}'.format(
self.op_state['test_suite_final_status']))
if self.op_state['test_suite_final_status'] == 'success':
return self.client.SUCCESS
elif self.op_state['test_suite_final_status'] == 'failure':
return self.client.FAILURE
elif self.op_state['test_suite_final_status'] == 'warning':
return self.client.WARNING
else:
return self.client.ERROR
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
class MyFlowService():
def __init__(self):
self.client = OperetoClient()
self.input = self.client.input
self.failed = False
def run(self):
try:
def _end_fake_parent_process(parent_pid, child_process_pids=[]):
if self.client.is_success(child_process_pids):
self.client.stop_process(parent_pid, 'success')
else:
self.failed = True
self.client.stop_process(parent_pid, 'failure')
## create a "fake" parent process
parent_1_pid = self.client.create_process(
'fake_parent_process', title='First commands group')
## attach some services to it as child processes
cmd_1_pid = self.client.create_process('shell_command',
title='df -k',
command='df -k',
pflow_id=parent_1_pid)
cmd_2_pid = self.client.create_process('shell_command',
title='netstat -rn',
command='netstat -rn',
pflow_id=parent_1_pid)
_end_fake_parent_process(parent_1_pid, [cmd_1_pid, cmd_2_pid])
time.sleep(10)
## create a second "fake" parent process
parent_2_pid = self.client.create_process(
'fake_parent_process', title='Second commands group')
## attach some services to it as child processes
cmd_3_pid = self.client.create_process('shell_command',
title='df -k',
command='df -k',
pflow_id=parent_2_pid)
cmd_4_pid = self.client.create_process('shell_command',
title='netstat -rn',
command='netstat -rn',
pflow_id=parent_2_pid)
_end_fake_parent_process(parent_2_pid, [cmd_3_pid, cmd_4_pid])
time.sleep(5)
## set final result
if self.failed:
return self.client.FAILURE
return self.client.SUCCESS
except:
traceback.format_exc()
return self.client.FAILURE
usage = "%prog -s service [-t title] [-a agent] [-v version] [--production]"
parser = OptionParser(usage=usage)
parser.add_option("-s", "--service", dest="service", help="service name.")
parser.add_option("-v", "--version", dest="version", help="service version specification (optional), for instance: 1.1.0-12.")
parser.add_option("-t", "--title", dest="title", help="process title (optional).")
parser.add_option("-a", "--agent", dest="agent", default='any', help="agent name (optional), default is 'any'.")
parser.add_option("--production", dest="production", action='store_true', default=False, help="Runs the default production version. The same as specifying [--version default]")
(options, args) = parser.parse_args()
if not options.service:
parser.error('service must be provided.')
return (options, args)
if __name__ == "__main__":
(options, args) = parse_options()
client = OperetoClient()
operations_mode = 'development'
if options.production or options.version:
operations_mode = 'production'
version=options.version
if not version:
version='default'
pid = client.create_process(service=options.service, service_version=version, title=options.title, agent=options.agent , mode=operations_mode)
if operations_mode=='production':
print 'A new process for service [%s] has been created: mode=%s, version=%s, pid=%s'%(options.service, operations_mode, options.version, pid)
else:
print 'A new development process for service [%s] has been created: pid=%s'%(options.service, pid)
def parse_options():
usage = "%prog -d path/to/service/directory [-s service] [-v version] [--production]"
parser = OptionParser(usage=usage)
parser.add_option("-d", "--dir", dest="dir", help="path to the service directory.")
parser.add_option("-s", "--service", dest="service", help="service name (optional), if not provided, directory name will be used.")
parser.add_option("-v", "--version", dest="version", help="service version specification, for instance: 1.1.0-12.")
parser.add_option("--production", dest="production", action='store_true', default=False, help="Uploads default version to production (the same as specifying: --version default)")
(options, args) = parser.parse_args()
if not options.dir:
parser.error('service directory must be provided.')
return (options, args)
if __name__ == "__main__":
(options, args) = parse_options()
client = OperetoClient()
zip_action_file = os.path.join(TEMP_DIR, str(uuid.uuid4())+'.action')
zipfolder(zip_action_file, options.dir)
operations_mode = 'development'
if options.production or options.version:
operations_mode = 'production'
version=options.version
if not version:
version='default'
client.upload_service_version(service_zip_file=zip_action_file+'.zip', mode=operations_mode, service_version=version, service_id=options.service)
if operations_mode=='production':
print 'Service production version [%s] deployed successfuly.'%version
else:
print 'Service development version deployed successfully.'
def __init__(self, **kwargs):
self.client = OperetoClient()
ServiceTemplate.__init__(self, **kwargs)
def opereto_client():
return OperetoClient()
