def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"pod_operation": {
"enum": ['create_pod', 'delete_pod']
},
"pod_name": {
"type": ["null", "string"]
},
"pod_template": {
"type": "object"
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'pod_operation', 'pod_template', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['pod_operation'] == 'delete_pod' and not self.input.get(
'pod_name'):
raise OperetoRuntimeError(
error='Pod name must be provided for this operation')
if self.input['pod_name'].startswith('opereto-worker-node'):
raise OperetoRuntimeError(
error=
'Pod name is invalid, this name is used for Opereto standard elastic workers. Please select a different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"deployment_operation": {
"enum": [
'create_statefulset', 'modify_statefulset',
'delete_statefulset', 'update_worker_image'
]
},
"deployment_name": {
"type": ["null", "string"]
},
"agent_java_config": {
"type": "string",
"minLength": 1
},
"agent_log_level": {
"enum": ['info', 'warn', 'error', 'fatal', 'debug']
},
"worker_config": {
"type": "string",
"minLength": 1
},
"agent_properties":
item_properties_scheme,
"required": [
'deployment_operation', 'agent_java_config',
'agent_log_level', 'worker_config', 'agent_properties'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
if self.input['deployment_name'] == 'opereto-worker-node':
raise OperetoRuntimeError(
error=
'Deployment name is invalid, this name is used for Opereto standard workers. Please insert different name.'
)
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def process_results():
tests_json = os.path.join(self.test_results_dir, 'tests.json')
if os.path.exists(tests_json):
with open(tests_json, 'r') as tf:
self.test_data = json.load(tf)
all_tests_completed=True
try:
validator = JsonSchemeValidator(self.test_data, self.tests_json_scheme)
validator.validate()
except Exception, e:
print 'Invalid tests json file: {}'.format(e)
return
if 'test_records' in self.test_data:
for test_record in self.test_data['test_records']:
self._modify_record(test_record)
if test_record['status'] not in self.result_keys:
all_tests_completed=False
if 'test_suite' in self.test_data and all_tests_completed:
if 'status' in self.test_data['test_suite']:
if self.test_data['test_suite']['status'] in self.result_keys:
self.end_of_test_suite=True
self.test_suite_final_status=self.test_data['test_suite']['status']
if 'links' in self.test_data['test_suite']:
self.suite_links = self.test_data['test_suite']['links']
def validate_input(self):
self._replace_empty_dict_with_none(
['task_service', 'test_parser_config'])
input_scheme = {
"type": "object",
"properties": {
"debug_mode": {
"type": "boolean"
},
"pre_task_services": included_services_scheme,
"valid_exit_codes": {
"type": "string"
},
"test_parser_config": services_block_scheme,
"test_results_directory": {
"type": ["string", "null"]
},
"keep_parser_running": {
"type": "integer",
"minimum": 5
},
"post_task_services": included_services_scheme
},
"required": [],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self._validate_input()
def _validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"docker_image": {
"type": ["string", "null"]
},
"docker_command": {
"type": "string",
"minLength": 1
},
"docker_env_params": {
"type": "object",
"patternProperties": {
default_variable_pattern: {
"type": "string"
}
}
},
},
"required": ['docker_command'],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"bucket_name": {
"type": "string",
"minLength": 1
},
"source_path": {
"type": "string",
"minLength": 1
},
"target_path": {
"type": ["string", "null"]
},
"presigned_url_expiry": {
"type": "integer"
},
"create_bucket": {
"type": "boolean"
},
"make_public": {
"type": "boolean"
},
"content_type": {
"type": "string"
},
"aws_access_key": {
"type": "string",
"minLength": 1
},
"aws_secret_key": {
"type": "string",
"minLength": 1
},
"required": [
'bucket_name', 'source_path', 'target_path',
'aws_access_key', 'aws_secret_key', 'content_type'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.source_path = self.input['source_path']
self.target_path = self.input['target_path'].lstrip('/')
self.session = boto3.session.Session(
aws_access_key_id=self.input['aws_access_key'],
aws_secret_access_key=self.input['aws_secret_key'])
self.s3_client = self.session.client('s3')
self.s3_res = self.session.resource('s3')
def validate_input(self):
self.task_exitcode = 3
self.task_output = {}
self.parent_pid = self.input['opereto_parent_flow_id'] or self.input[
'pid']
self.debug_mode = self.input['debug_mode']
self.listener_pid = None
self.parser_pid = None
self.listener_results_dir = os.path.join(
self.input['opereto_workspace'], 'opereto_listener_results')
self.parser_results_directory = os.path.join(
self.input['opereto_workspace'], 'opereto_parser_results')
self.test_results_directory = self.input['test_results_directory']
self.task_output_json = os.path.join(self.input['opereto_workspace'],
'opereto_task_output.json')
self._replace_empty_dict_with_none(
['task_service', 'test_parser_config'])
input_scheme = {
"type": "object",
"properties": {
"debug_mode": {
"type": "boolean"
},
"pre_task_services": included_services_scheme,
"test_parser_config": services_block_scheme,
"test_results_directory": {
"type": ["string", "null"]
},
"keep_parser_running": {
"type": "integer",
"minimum": 5
},
"post_task_services": included_services_scheme
},
"required": [],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self._validate_input()
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"cf_stack_name": {
"type": "string",
"minLength": 1
},
"aws_access_key": {
"type": "string",
"minLength": 1
},
"aws_secret_key": {
"type": "string",
"minLength": 1
},
"aws_region": {
"type": "string",
"minLength": 1
},
"required": [
'cf_stack_name', 'aws_access_key', 'aws_secret_key',
'aws_region'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.cf_stack_name = self.input['cf_stack_name']
self.aws_access_key = self.input['aws_access_key']
self.aws_secret_key = self.input['aws_secret_key']
self.aws_region = self.input['aws_region']
self.session = boto3.session.Session(
aws_access_key_id=self.input['aws_access_key'],
aws_secret_access_key=self.input['aws_secret_key'])
self.cf_conn = self.session.client(
'cloudformation', region_name=self.input['aws_region'])
print('Connected.')
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"bucket_name": {
"type": "string",
"minLength": 1
},
"source_path": {
"type": "string",
"minLength": 1
},
"target_path": {
"type": "string",
"minLength": 1
},
"aws_access_key": {
"type": "string",
"minLength": 1
},
"aws_secret_key": {
"type": "string",
"minLength": 1
},
"required": [
'bucket_name', 'source_path', 'target_path',
'aws_access_key', 'aws_secret_key'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.source_path = self.input['source_path']
self.target_path = self.input['target_path']
self.session = boto3.session.Session(
aws_access_key_id=self.input['aws_access_key'],
aws_secret_access_key=self.input['aws_secret_key'])
self.s3_client = self.session.client('s3')
self.s3_res = self.session.resource('s3')
def process_results():
try:
tests_json = os.path.join(self.test_results_dir, 'tests.json')
if os.path.exists(tests_json):
with open(tests_json, 'r') as tf:
try:
self.test_data = json.load(tf)
self.op_state['test_data'] = self.test_data
try:
validator = JsonSchemeValidator(
self.test_data, self.tests_json_scheme)
validator.validate()
except Exception as e:
print('Invalid tests json file: {}'.format(e))
return
if 'test_records' in self.test_data:
for test_record in self.test_data[
'test_records']:
self._modify_record(test_record)
if 'test_suite' in self.test_data:
if 'status' in self.test_data['test_suite']:
self.op_state[
'test_suite_final_status'] = self.test_data[
'test_suite']['status']
if 'links' in self.test_data['test_suite']:
self.suite_links = self.test_data[
'test_suite']['links']
self.op_state[
'test_suite'] = self.suite_links
finally:
try:
self._save_state(self.op_state)
except Exception as e:
print(e)
finally:
self.end_of_test_suite = self.client.get_process_property(
name='stop_listener_code')
if self.debug_mode:
print('[DEBUG] content of tests.json: {}'.format(
json.dumps(self.test_data)))
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"deployment_name": {
"type": "string",
"minLength": 1
},
"gcp_access_credentials": {
"type": "object"
},
"required": ['deployment_name', 'gcp_access_credentials'],
"additionalProperties": True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.deployment_name = self.input['deployment_name']
self._print_step_title('Connecting to GCP..')
current_credential_file = os.path.join(self.input['opereto_workspace'],
'client_secret.json')
with open(current_credential_file, 'w') as cf:
cf.write(json.dumps(self.input['gcp_access_credentials'],
indent=4))
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = current_credential_file
credentials = gce.AppAssertionCredentials(
scope='https://www.googleapis.com/auth/cloud-platform')
self.gcp_http_handler = credentials.authorize(httplib2.Http())
self.gcp_deploy_manager = build('deploymentmanager', 'v2')
print 'Connected.'
def validate_input(self):
input_scheme = {
"type": "object",
"properties" : {
"deployment_name": {
"type" : "string",
"minLength": 1
},
"gcp_project_id": {
"type": "string",
"minLength": 1
},
"deployment_template": {
"type" : ["string", "null"]
},
"deployment_import_templates": {
"type": ["string", "null"]
},
"deployment_parameters": {
"type": ["object", "null"]
},
"opereto_core_tools": {
"type" : "boolean"
},
"opereto_container_tools": {
"type" : "boolean"
},
"disable_rollback": {
"type": "boolean"
},
"gcp_access_credentials": {
"type": "object"
},
"agent_package_url": {
"type": "object",
"properties": {
"windows": {
"type": "string",
"minLength": 1
},
"linux": {
"type": "string",
"minLength": 1
}
},
"required": ['windows', 'linux'],
"additionalProperties": True
},
"required": ['gcp_project_id', 'deployment_template', 'gcp_access_credentials'],
"additionalProperties": True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.install_core_tools = self.input['install_core_tools']
self.install_container_tools = self.input['install_container_tools']
self.agent_valid_os = ['linux', 'windows']
self.deployment_name = self.input['deployment_name']
if self.install_container_tools and not self.install_core_tools:
raise Exception('Opereto container tools is dependant on opereto core tools. Please check the "install_core_tools" checkbox too.')
source_user = self.client.input['opereto_originator_username']
agent_user = self.client.input['opereto_user']
self.users = [source_user, agent_user]
self.owners = [source_user, agent_user]
def linux_user_data(agent_name):
agent_install_command = './install.sh -h {} -t {} -n {}'.format(self.input['opereto_host'], self.token,
agent_name)
if get_opereto_major_release(self.client)<3:
agent_install_command = './install.sh -b {} -u {} -p {} -n {}'.format(self.input['opereto_host'], agent_user, self.input['opereto_password'], agent_name)
data = """
items:
- key: startup-script
value: |
#! /bin/bash
curl -O {}
tar -zxvf opereto-agent-latest.tar.gz
cd opereto-agent-latest
sudo chmod 777 -R *
{}""".format(self.input['agent_package_url']['linux'], agent_install_command)
return data
def windows_user_data(agent_name):
data = """
items:
- key: startup-script
value: |
<powershell>
Add-Type -AssemblyName System.IO.Compression.FileSystem
function Unzip
{{
param([string]$zipfile, [string]$outpath)
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
}}
$MyDir = "c:"
$filename = Join-Path -Path $MyDir -ChildPath "opereto-agent-latest.zip"
$WebClient = New-Object System.Net.WebClient
$WebClient.DownloadFile("{}", "$filename")
Unzip "$MyDir\opereto-agent-latest.zip" "$MyDir\opereto"
cd "$MyDir\opereto\opereto-agent-latest"
./opereto-install.bat {} {} {} javaw
./opereto-start.bat
Remove-Item $filename
</powershell>
<persist>true</persist>""".format(self.input['agent_package_url']['windows'], self.input['opereto_host'], self.token, agent_name)
return data
def _add_agent_installation(json_template):
if json_template.get('resources'):
for resource_data in json_template['resources']:
if resource_data["type"]=="compute.v1.instance":
agent_os=None
agent_name=None
agent_display_name=None
agent_description=None
agent_id_found=False
if resource_data['properties'].get("labels"):
for key, value in resource_data['properties']["labels"].items():
if key=='opereto-agent-os':
agent_os=value
elif key=='opereto-agent-id':
agent_id_found=True
agent_name=value
elif key=='opereto-agent-name':
agent_display_name=value
elif key=='opereto-agent-desc':
agent_description=value
json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-id'] = agent_name.lower() ## match gcp label value policy
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-name']
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-desc']
if agent_os:
if agent_os not in self.agent_valid_os:
raise OperetoRuntimeError('OperetoAgentOs must be one of the following: {}'.format(str(self.agent_valid_os)))
if not agent_name:
agent_name = 'agent'+str(uuid.uuid4())[:10]
else:
try:
JsonSchemeValidator(agent_name, default_variable_name_scheme).validate()
except Exception as e:
raise OperetoRuntimeError('Invalid agent identifier: {}'.format(str(e)))
if agent_display_name:
try:
JsonSchemeValidator(agent_display_name, default_entity_name_scheme).validate()
except Exception as e:
raise OperetoRuntimeError('Invalid agent agent_install_commandname: {}'.format(str(e)))
if agent_description:
try:
JsonSchemeValidator(agent_description, default_entity_description_scheme).validate()
except Exception as e:
raise OperetoRuntimeError('Invalid agent description: {}'.format(str(e)))
if agent_os=='windows':
agent_data = windows_user_data(agent_name)
else:
agent_data = linux_user_data(agent_name)
self.agent_data_map[agent_name]=agent_data
## currently override user data, add fix to handle addition to existing user data
##
##
json_template['resources'][json_template['resources'].index(resource_data)]['properties']["metadata"] = agent_name+'-meta-placeholder'
self.agents[agent_name]={
'agent_display_name': agent_display_name,
'agent_description': agent_description
}
template_in_yaml = yaml.dump(json_template)
for agent_name, agent_data in self.agent_data_map.items():
new_template = template_in_yaml.replace(agent_name+'-meta-placeholder', agent_data)
template_in_yaml = new_template
return template_in_yaml
self.deployment_template = _add_agent_installation(yaml.load(self.input['deployment_template'], Loader=yaml.SafeLoader))
self.deployment_import_templates = []
if self.input['deployment_import_templates']:
for name, content in yaml.load(self.input['deployment_import_templates'], Loader=yaml.SafeLoader).items():
content = _add_agent_installation(content)
entry = {
"name": name,
"content": """{}
""".format(content)
}
self.deployment_import_templates.append(entry)
self.deployment_exist = False
self._print_step_title('Connecting to GCP..')
current_credential_file = os.path.join(self.input['opereto_workspace'], 'client_secret.json')
with open(current_credential_file, 'w') as cf:
cf.write(json.dumps(self.input['gcp_access_credentials'], indent=4))
os.environ['GOOGLE_APPLICATION_CREDENTIALS']=current_credential_file
credentials = gce.AppAssertionCredentials(
scope='https://www.googleapis.com/auth/cloud-platform'
)
self.gcp_http_handler = credentials.authorize(httplib2.Http())
self.gcp_compute_manager = build('compute', 'v1')
self.gcp_deploy_manager = build('deploymentmanager', 'v2')
print('Connected.')
def _add_agent_installation(json_template):
if json_template.get('resources'):
for resource_data in json_template['resources']:
if resource_data["type"] == "compute.v1.instance":
agent_os = None
agent_name = None
agent_display_name = None
agent_description = None
agent_id_found = False
if resource_data['properties'].get("labels"):
for key, value in resource_data['properties'][
"labels"].items():
if key == 'opereto-agent-os':
agent_os = value
elif key == 'opereto-agent-id':
agent_id_found = True
agent_name = value
elif key == 'opereto-agent-name':
agent_display_name = value
elif key == 'opereto-agent-desc':
agent_description = value
json_template['resources'][
json_template['resources'].index(
resource_data)]['properties']["labels"][
'opereto-agent-id'] = agent_name.lower(
) ## match gcp label value policy
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-name']
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-desc']
if agent_os:
if agent_os not in self.agent_valid_os:
raise OperetoRuntimeError(
'OperetoAgentOs must be one of the following: {}'
.format(str(self.agent_valid_os)))
if not agent_name:
agent_name = 'agent' + str(uuid.uuid4())[:10]
else:
try:
JsonSchemeValidator(
agent_name,
default_variable_name_scheme).validate(
)
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent identifier: {}'.format(
str(e)))
if agent_display_name:
try:
JsonSchemeValidator(
agent_display_name,
default_entity_name_scheme).validate()
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent name: {}'.format(
str(e)))
if agent_description:
try:
JsonSchemeValidator(
agent_description,
default_entity_description_scheme
).validate()
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent description: {}'.format(
str(e)))
if agent_os == 'windows':
agent_data = windows_user_data(agent_name)
else:
agent_data = linux_user_data(agent_name)
self.agent_data_map[agent_name] = agent_data
## currently override user data, add fix to handle addition to existing user data
##
##
json_template['resources'][
json_template['resources'].
index(resource_data)]['properties'][
"metadata"] = agent_name + '-meta-placeholder'
self.agents[agent_name] = {
'agent_display_name': agent_display_name,
'agent_description': agent_description
}
def _validate_input(self):
self.pod_template = self.input['pod_template']
self.output_file_path = self.input['output_file_path']
input_scheme = {
"type": "object",
"properties": {
"pod_config_files": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"src": {
"type": "string"
},
"target": {
"type": "string"
}
}
},
"minItems": 0
},
"parser_worker_image": {
"type": "string"
},
"pod_template": {
"type": "object"
},
"output_file_path": {
"type": ["string", "null"]
},
"required": ['pod_template'],
"additionalProperties": True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
try:
if self.pod_template["metadata"]["name"].startswith('opereto'):
raise OperetoRuntimeError(
error=
'Pod names containing the prefix [opereto] are reserved names. Please select a different name.'
)
except Exception as e:
raise OperetoRuntimeError(
error='Could not identify pod name: {}'.format(str(e)))
if self.input['pass_opereto_credentials']:
containers = self.pod_template["spec"]["containers"]
for container in containers:
if not container.get('env'):
container['env'] = []
container['env'] += [{
"name": "opereto_host",
"valueFrom": {
"secretKeyRef": {
"key": "OPERETO_HOST",
"name": "worker-config"
}
}
}, {
"name": "opereto_token",
"valueFrom": {
"secretKeyRef": {
"key": "OPERETO_TOKEN",
"name": "worker-config"
}
}
}]
self.pod_template["spec"]["containers"] = containers
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"deployment_name": {
"type": "string",
"minLength": 1
},
"gcp_project_id": {
"type": "string",
"minLength": 1
},
"deployment_template": {
"type": ["string", "null"]
},
"deployment_import_templates": {
"type": ["string", "null"]
},
"deployment_parameters": {
"type": ["object", "null"]
},
"opereto_core_tools": {
"type": "boolean"
},
"opereto_container_tools": {
"type": "boolean"
},
"disable_rollback": {
"type": "boolean"
},
"gcp_access_credentials": {
"type": "object"
},
"agent_package_url": {
"type": "object",
"properties": {
"windows": {
"type": "string",
"minLength": 1
},
"linux": {
"type": "string",
"minLength": 1
}
},
"required": ['windows', 'linux'],
"additionalProperties": True
},
"required": [
'gcp_project_id', 'deployment_template',
'gcp_access_credentials'
],
"additionalProperties":
True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.install_core_tools = self.input['install_core_tools']
self.install_container_tools = self.input['install_container_tools']
self.agent_valid_os = ['linux', 'windows']
self.deployment_name = self.input['deployment_name']
if self.install_container_tools and not self.install_core_tools:
raise Exception, 'Opereto container tools is dependant on opereto core tools. Please check the "install_core_tools" checkbox too.'
source_user = self.client.input['opereto_originator_username']
self.users = [source_user]
self.owners = [source_user]
def linux_user_data(agent_name):
data = """
items:
- key: startup-script
value: |
#! /bin/bash
curl -O {}
tar -zxvf opereto-agent-latest.tar.gz
cd opereto-agent-latest
sudo chmod 777 -R *
./install.sh -b {} -u {} -p {} -n {}""".format(
self.input['agent_package_url']['linux'],
self.input['opereto_host'], self.input['opereto_user'],
self.input['opereto_password'], agent_name)
return data
def windows_user_data(agent_name):
data = """
items:
- key: startup-script
value: |
<powershell>
Add-Type -AssemblyName System.IO.Compression.FileSystem
function Unzip
{
param([string]$zipfile, [string]$outpath)
[System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
}
$MyDir = "c:"
$filename = Join-Path -Path $MyDir -ChildPath "opereto-agent-latest.zip"
$WebClient = New-Object System.Net.WebClient
$WebClient.DownloadFile("%s", "$filename")
Unzip "$MyDir\opereto-agent-latest.zip" "$MyDir\opereto"
cd "$MyDir\opereto\opereto-agent-latest"
./opereto-install.bat %s %s "%s" %s javaw
./opereto-start.bat
Remove-Item $filename
</powershell>
<persist>true</persist>""" % (
self.input['agent_package_url']['windows'],
self.input['opereto_host'], self.input['opereto_user'],
self.input['opereto_password'], agent_name)
return data
def _add_agent_installation(json_template):
if json_template.get('resources'):
for resource_data in json_template['resources']:
if resource_data["type"] == "compute.v1.instance":
agent_os = None
agent_name = None
agent_display_name = None
agent_description = None
agent_id_found = False
if resource_data['properties'].get("labels"):
for key, value in resource_data['properties'][
"labels"].items():
if key == 'opereto-agent-os':
agent_os = value
elif key == 'opereto-agent-id':
agent_id_found = True
agent_name = value
elif key == 'opereto-agent-name':
agent_display_name = value
elif key == 'opereto-agent-desc':
agent_description = value
json_template['resources'][
json_template['resources'].index(
resource_data)]['properties']["labels"][
'opereto-agent-id'] = agent_name.lower(
) ## match gcp label value policy
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-name']
# del json_template['resources'][json_template['resources'].index(resource_data)]['properties']["labels"]['opereto-agent-desc']
if agent_os:
if agent_os not in self.agent_valid_os:
raise OperetoRuntimeError(
'OperetoAgentOs must be one of the following: {}'
.format(str(self.agent_valid_os)))
if not agent_name:
agent_name = 'agent' + str(uuid.uuid4())[:10]
else:
try:
JsonSchemeValidator(
agent_name,
default_variable_name_scheme).validate(
)
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent identifier: {}'.format(
str(e)))
if agent_display_name:
try:
JsonSchemeValidator(
agent_display_name,
default_entity_name_scheme).validate()
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent name: {}'.format(
str(e)))
if agent_description:
try:
JsonSchemeValidator(
agent_description,
default_entity_description_scheme
).validate()
except Exception, e:
raise OperetoRuntimeError(
'Invalid agent description: {}'.format(
str(e)))
if agent_os == 'windows':
agent_data = windows_user_data(agent_name)
else:
agent_data = linux_user_data(agent_name)
self.agent_data_map[agent_name] = agent_data
## currently override user data, add fix to handle addition to existing user data
##
##
json_template['resources'][
json_template['resources'].
index(resource_data)]['properties'][
"metadata"] = agent_name + '-meta-placeholder'
self.agents[agent_name] = {
'agent_display_name': agent_display_name,
'agent_description': agent_description
}
def validate_input(self):
## set and validate users
source_user = self.client.input['opereto_originator_username']
self.users = [source_user]
self.owners = [source_user]
try:
if self.input['additional_users'] is not None:
additional_users = self.input['additional_users'].replace(
" ", "").split(',')
except:
additional_users = []
try:
if self.input['additional_owners'] is not None:
additional_owners = self.input['additional_owners'].replace(
" ", "").split(',')
except:
additional_owners = []
self.users = [source_user] + list(
set([x for x in additional_users if x]))
self.owners = [source_user] + list(
set([x for x in additional_owners if x]))
all_users = self.users + self.owners
existing_users = self.client.search_users()
existing_user_ids = []
for existing in existing_users:
existing_user_ids.append(existing['id'])
for user in all_users:
if user != '' and user not in existing_user_ids:
print >> sys.stderr, 'User {} does not exist'.format(user)
return self.client.FAILURE
self.agent_id = self.input['agent_identifier'] or 'dockereto-' + str(
uuid.uuid4())[:12]
## agent id
validator = JsonSchemeValidator(self.agent_id,
default_variable_name_scheme)
validator.validate()
## agent name
validator = JsonSchemeValidator(self.input['agent_name'],
default_entity_name_scheme)
validator.validate()
## agent description
validator = JsonSchemeValidator(self.input['agent_description'],
default_entity_description_scheme)
validator.validate()
## agent_properties
if self.input['agent_properties']:
validator = JsonSchemeValidator(self.input['agent_properties'],
item_properties_scheme)
validator.validate()
## post_operations
if self.input['post_operations']:
validator = JsonSchemeValidator(self.input['post_operations'],
included_services_scheme)
validator.validate()
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"test_results_path": {
"type": "string",
"minLength": 1
},
"parent_pid": {
"type": "string"
},
"listener_frequency": {
"type": "integer",
"minValue": 1
},
"debug_mode": {
"type": "boolean"
}
},
"required": ['listener_frequency', 'test_results_path'],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.parent_pid = self.input['parent_pid'] or self.input['pid']
self.test_results_dir = self.input['test_results_path']
self.debug_mode = self.input['debug_mode']
self.result_keys = process_result_keys
self.status_keys = process_status_keys
self.tests_json_scheme = {
"type": "object",
"properties": {
"test_suite": {
"type": "object",
"properties": {
"links": {
"type": "array"
},
"status": {
"enum": self.status_keys
}
}
},
"test_records": {
"type":
"array",
"items": [{
"type": "object",
"properties": {
"testname": default_entity_name_scheme,
"status": {
"enum": self.status_keys
},
"title": default_entity_name_scheme,
"links": {
"type":
"array",
"items": [{
"type": "object",
"properties": {
"url": {
"type": "string"
},
"name": {
"type": "string"
}
}
}]
}
},
"required": ['testname', 'status'],
"additionalProperties": True
}]
}
},
"additionalProperties": True
}
self.end_of_test_suite = None
self.test_data = {}
self.suite_links = []
self._state = {}
def validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"cf_stack_name": {
"type": "string",
"minLength": 1
},
"cf_template": {
"type": ["object", "null"]
},
"cf_template_url": {
"type": ["string", "null"]
},
"opereto_core_tools": {
"type": "boolean"
},
"opereto_container_tools": {
"type": "boolean"
},
"agent_package_url": {
"type": "object",
"properties": {
"windows": {
"type": "string",
"minLength": 1
},
"linux": {
"type": "string",
"minLength": 1
}
},
"required": ['windows', 'linux'],
"additionalProperties": True
},
"disable_rollback": {
"type": "boolean"
},
"cf_capabilities": {
"type": ["string", "null"]
},
"cf_parameters": {
"type": ["object", "null"]
},
"cf_tags": {
"type": ["object", "null"]
},
"cf_globals": {
"type": ["object", "null"]
},
"aws_access_key": {
"type": "string",
"minLength": 1
},
"aws_secret_key": {
"type": "string",
"minLength": 1
},
"aws_region": {
"type": "string",
"minLength": 1
},
"required": ['aws_access_key', 'aws_secret_key', 'aws_region'],
"additionalProperties": True
}
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.cf_template = self.input['cf_template']
self.cf_template_url = self.input['cf_template_url']
if not self.cf_template and not self.cf_template_url:
raise Exception(
'Cloud formation template json (or an s3 url of a template) must be provided.'
)
self.install_core_tools = self.input['install_core_tools']
self.install_container_tools = self.input['install_container_tools']
self.agent_valid_os = ['linux', 'windows']
self.cf_capabilities = self.input['cf_capabilities'].split(',')
self.cf_globals = self.input['cf_globals']
if self.install_container_tools and not self.install_core_tools:
raise Exception(
'Opereto container tools is dependant on opereto core tools. Please check the "install_core_tools" checkbox too.'
)
source_user = self.client.input['opereto_originator_username']
agent_user = self.client.input['opereto_user']
self.users = [source_user, agent_user]
self.owners = [source_user, agent_user]
def windows_user_data(agent_name):
data = [
"<powershell>",
"Add-Type -AssemblyName System.IO.Compression.FileSystem",
"function Unzip", "{",
" param([string]$zipfile, [string]$outpath)",
" [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)",
"}", "$MyDir = \"c:\"",
"$filename = Join-Path -Path $MyDir -ChildPath \"opereto-agent-latest.zip\"",
"$WebClient = New-Object System.Net.WebClient",
"$WebClient.DownloadFile(\"{}\", \"$filename\")".format(
self.input['agent_package_url']['windows']),
"Unzip \"$MyDir\opereto-agent-latest.zip\" \"$MyDir\opereto\"",
"cd \"$MyDir\opereto\opereto-agent-latest\"",
"./opereto-install.bat {} {} {} javaw".format(
self.input['opereto_host'], self.token, agent_name),
"./opereto-start.bat", "Remove-Item $filename",
"</powershell>", "<persist>true</persist>"
]
return data
def linux_user_data(agent_name):
data = [
"#!/bin/bash -x",
"exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1",
"sed --in-place /requiretty/d /etc/sudoers", "cd /tmp",
"curl -O {}".format(self.input['agent_package_url']['linux']),
"tar -zxvf opereto-agent-latest.tar.gz",
"cd opereto-agent-latest", "sudo chmod 777 -R *",
"./install.sh -h {} -t {} -n {}".format(
self.input['opereto_host'], self.token, agent_name)
]
return data
if "Resources" in self.cf_template:
for resource_name, resource_data in self.cf_template[
'Resources'].items():
if resource_data["Type"] == "AWS::EC2::Instance":
agent_os = None
agent_name = None
agent_display_name = None
agent_description = None
agent_id_found = False
for tag in resource_data["Properties"]["Tags"]:
if tag['Key'] == 'OperetoAgentOs':
agent_os = tag['Value']
elif tag['Key'] == 'OperetoAgentId':
agent_id_found = True
agent_name = tag['Value']
elif tag['Key'] == 'OperetoAgentName':
agent_display_name = tag['Value']
elif tag['Key'] == 'OperetoAgentDesc':
agent_description = tag['Value']
if agent_os:
if agent_os not in self.agent_valid_os:
raise OperetoRuntimeError(
'OperetoAgentOs must be one of the following: {}'
.format(str(self.agent_valid_os)))
if not agent_name:
agent_name = 'agent' + str(uuid.uuid4())[:10]
else:
try:
JsonSchemeValidator(
agent_name,
default_variable_name_scheme).validate()
except Exception as e:
raise OperetoRuntimeError(
'Invalid agent identifier: {}'.format(
str(e)))
if agent_display_name:
try:
JsonSchemeValidator(
agent_display_name,
default_entity_name_scheme).validate()
except Exception as e:
raise OperetoRuntimeError(
'Invalid agent name: {}'.format(str(e)))
if agent_description:
try:
JsonSchemeValidator(
agent_description,
default_entity_description_scheme
).validate()
except Exception as e:
raise OperetoRuntimeError(
'Invalid agent description: {}'.format(
str(e)))
if agent_os == 'windows':
agent_data = windows_user_data(agent_name)
else:
agent_data = linux_user_data(agent_name)
## currently override user data, add fix to handle addition to existing user data
##
##
self.cf_template["Resources"][resource_name][
"Properties"]["UserData"] = {
"Fn::Base64": {
"Fn::Join": ["\n", agent_data]
}
}
if not agent_id_found:
self.cf_template["Resources"][resource_name][
"Properties"]["Tags"].append({
"Key":
"OperetoAgentId",
"Value":
agent_name
})
self.agents[agent_name] = {
'cf_stack_name': self.input['cf_stack_name'],
'agent_display_name': agent_display_name,
'agent_description': agent_description
}
self._print_step_title('Connecting to AWS..')
self.session = boto3.session.Session(
aws_access_key_id=self.input['aws_access_key'],
aws_secret_access_key=self.input['aws_secret_key'])
self.ec2_conn = self.session.client(
'ec2', region_name=self.input['aws_region'])
self.cf_conn = self.session.client(
'cloudformation', region_name=self.input['aws_region'])
print('Connected.')
def _validate_input(self):
input_scheme = {
"type": "object",
"properties": {
"task_services": included_services_scheme,
"worker_config": {
"type": 'object'
}
},
"required": ['worker_config', 'task_services'],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input, input_scheme)
validator.validate()
self.worker_agent_id = None
self.remove_worker = False
if not self.input['worker_config'].get('agent_id'):
worker_config_service = services_block_scheme
worker_config_service['properties']['agent_id_property'] = {
"type": "string",
"pattern": default_variable_pattern
}
worker_config_service['required'].append('agent_id_property')
if not self.input['worker_config'].get('setup') or not self.input[
'worker_config'].get('teardown'):
raise OperetoRuntimeError(
error=
'Worker configuration is invalid. Either an existing agent name or setup/teardown worker services must be provided. See service documentation for more details.'
)
else:
scheme = {
"type": "object",
"properties": {
"setup": worker_config_service,
"teardown": worker_config_service,
},
"required": ['setup', 'teardown'],
"additionalProperties": True
}
validator = JsonSchemeValidator(self.input['worker_config'],
scheme)
validator.validate()
else:
scheme = {
"type": "object",
"properties": {
"agent_id": {
"type": "string",
"pattern": default_variable_pattern
},
"required": ['agent_id'],
"additionalProperties": True
}
}
validator = JsonSchemeValidator(self.input['worker_config'],
scheme)
validator.validate()
self.worker_agent_id = self.input['worker_config'].get('agent_id')