def handle_delivery(message):
logger.info("Got: " + str(message))
try:
message = message.decode()
except (UnicodeDecodeError, AttributeError):
pass
parsed_json_message = json.loads(message)
owner = parsed_json_message['owner']
tosca_file_name = 'tosca_template'
tosca_template_dict = parsed_json_message['toscaTemplate']
tosca_template_path = save_tosca_template(tosca_template_dict)
tosca_helper = ToscaHelper(sure_tosca_base_url, tosca_template_path)
# nodes_to_deploy = tosca_helper.get_application_nodes()
nodes_pairs = tosca_helper.get_deployment_node_pairs()
deployService = DeployService(semaphore_base_url=semaphore_base_url, semaphore_username=semaphore_username,
semaphore_password=semaphore_password)
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000))
response["creationDate"] = output_current_milli_time
logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response))
return json.dumps(response)
def test_deploy_service(self):
parsed_json_message = self.get_request_message(
'https://raw.githubusercontent.com/qcdis-sdia/sdia-deployer/master/sample_requests/deploy_request_mog.json'
)
tosca_template_path = self.get_tosca_template_path(parsed_json_message)
# owner = parsed_json_message['owner']
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
self.assertIsNotNone(tosca_helper.doc_id)
nodes_to_deploy = tosca_helper.get_application_nodes()
self.assertIsNotNone(nodes_to_deploy)
nodes_pairs = tosca_helper.get_deployment_node_pipeline()
self.assertIsNotNone(nodes_pairs)
username = '******'
deployService = DeployService(
semaphore_base_url=semaphore_base_url,
semaphore_username=username,
semaphore_password='******',
vms=tosca_helper.get_vms())
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
def test(self):
parsed_json_message = self.get_request_message()
tosca_template_path = self.get_tosca_template_path(parsed_json_message)
# owner = parsed_json_message['owner']
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
self.assertIsNotNone(tosca_helper.doc_id)
nodes_to_deploy = tosca_helper.get_application_nodes()
self.assertIsNotNone(nodes_to_deploy)
nodes_pairs = tosca_helper.get_deployment_node_pipeline()
self.assertIsNotNone(nodes_pairs)
username = '******'
deployService = DeployService(
polemarch_base_url=polemarch_base_url,
polemarch_username=username,
polemarch_password='******',
semaphore_base_url=semaphore_base_url,
semaphore_username=username,
semaphore_password='******',
vms=tosca_helper.get_vms())
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
def test_deployer(self):
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
f = open('../sample_requests/deploy_request_mog.json', )
parsed_json_message = json.load(f)
tosca_template_path = self.get_tosca_template_path(
parsed_json_message)
tosca_template_dict = parsed_json_message['toscaTemplate']
if 'workflows' in tosca_template_dict['topology_template']:
return self.awx(tosca_template_dict=tosca_template_dict,
tosca_template_path=tosca_template_path)
return self.semaphore(tosca_template_dict=tosca_template_dict,
tosca_template_path=tosca_template_path)
def awx(self, tosca_template_path=None, tosca_template_dict=None):
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
if tosca_service_is_up:
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
node_templates = tosca_template_dict['topology_template'][
'node_templates']
awx = AWXService(api_url=awx_base_url,
username=awx_username,
password=awx_password)
topology_template_workflow_steps = {}
for tosca_node_name in node_templates:
tosca_node = node_templates[tosca_node_name]
logger.info('Resolving function values for: ' +
tosca_node_name)
tosca_node = tosca_helper.resolve_function_values(tosca_node)
logger.info('Creating workflow steps for: ' + tosca_node_name)
node_workflow_steps = awx.create_workflow_templates(tosca_node)
topology_template_workflow_steps.update(node_workflow_steps)
workflows = tosca_helper.get_workflows()
if workflows:
launched_ids = []
for workflow_name in workflows:
workflow = workflows[workflow_name]
description = None
if 'description' in workflow:
description = workflow['description']
wf_ids = awx.create_workflow(description=description,
workflow_name=workflow_name)
logger.info('Created workflow with ID: ' + str(wf_ids[0]))
workflow_node_ids = awx.create_dag(
workflow_id=wf_ids[0],
tosca_workflow=workflow,
topology_template_workflow_steps=
topology_template_workflow_steps,
workflow_name=workflow_name)
logger.info('Added nodes to workflow')
for wf_id in wf_ids:
wf_job_ids = awx.launch(wf_id)
launched_ids += wf_job_ids
def semaphore(self, tosca_template_path=None, tosca_template_dict=None):
tosca_helper = ToscaHelper(sure_tosca_base_url, tosca_template_path)
nodes = tosca_helper.get_application_nodes()
# nodes = tosca_helper.get_deployment_node_pipeline()
deployService = DeployService(semaphore_base_url=semaphore_base_url,
semaphore_username=semaphore_username,
semaphore_password=semaphore_password,
vms=tosca_helper.get_vms())
try:
for node in nodes:
updated_node = deployService.deploy(node)
if isinstance(updated_node, list):
for node in updated_node:
tosca_template_dict = tosca_helper.set_node(
node, tosca_template_dict)
# logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
else:
tosca_template_dict = tosca_helper.set_node(
updated_node, tosca_template_dict)
# logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000))
response["creationDate"] = output_current_milli_time
logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response))
return json.dumps(response)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
def test(self):
tosca_template_dict = self.get_tosca_from_url(
'https://raw.githubusercontent.com/qcdis-sdia/sdia-tosca/master/examples/TIC_planed.yaml'
)
tmp_path = tempfile.mkdtemp()
tosca_template_path = tmp_path + os.path.sep + 'toscaTemplate.yml'
with open(tosca_template_path, 'w') as outfile:
yaml.dump(tosca_template_dict, outfile, default_flow_style=False)
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
if tosca_service_is_up:
node_templates = tosca_template_dict['topology_template'][
'node_templates']
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
for tosca_node_name in node_templates:
tosca_node = node_templates[tosca_node_name]
tosca_node = tosca_helper.resolve_function_values(tosca_node)
interfaces = tosca_node['interfaces']
for interface_name in interfaces:
interface_ancestors = tosca_helper.get_interface_ancestors(
interface_name)
self.assertIsNotNone(interface_ancestors)
def test_inventory(self):
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
parsed_json_message = self.get_request_message()
tosca_template_path = self.get_tosca_template_path(
parsed_json_message)
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
nodes_pairs = tosca_helper.get_deployment_node_pipeline()
vms = tosca_helper.get_vms()
def test_inventory(self):
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
parsed_json_message = self.get_request_message(
'https://raw.githubusercontent.com/qcdis-sdia/sdia-deployer/master/sample_requests/deploy_request_mog.json'
)
tosca_template_path = self.get_tosca_template_path(
parsed_json_message)
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
nodes_pairs = tosca_helper.get_deployment_node_pipeline()
vms = tosca_helper.get_vms()
def encrypt_credentials(tosca_template_dict):
logger.info('Encrypting credentials.')
node_templates = tosca_template_dict['topology_template']['node_templates']
enc_key = bytes(secret, 'utf-8')
for node_template_name in node_templates:
node_template = node_templates[node_template_name]
credentials = ToscaHelper.extract_credentials_from_node(node_template)
if credentials:
for credential in credentials:
if 'token' not in credential:
# This is a tmp fix for the tosca parser. The tosca.datatypes.Credential which requires token
credential['token'] = 'dG9rZW4K'
if 'protocol' in credential and credential['protocol'] == 'ssh':
continue
if 'token' in credential:
token = credential['token']
credential['token'] = encrypt(token, enc_key)
if 'keys' in credential:
keys = credential['keys']
for key_name in keys:
token = keys[key_name]
keys[key_name] = encrypt(token, enc_key)
return tosca_template_dict
def decrypt_credentials(tosca_template_dict):
logger.info('Decrypting credentials.')
node_templates = tosca_template_dict['topology_template']['node_templates']
enc_key = bytes(secret, 'utf-8')
for node_template_name in node_templates:
node_template = node_templates[node_template_name]
if node_template['type'] == 'tosca.nodes.QC.VM.Compute':
continue
credentials = ToscaHelper.extract_credentials_from_node(node_template)
if credentials:
for credential in credentials:
if 'protocol' in credential and credential['protocol'] == 'ssh':
continue
if 'token' in credential:
token = credential['token']
credential['token'] = decrypt(token, enc_key)
if 'keys' in credential:
keys = credential['keys']
for key_name in keys:
token = keys[key_name]
keys[key_name] = decrypt(token, enc_key)
return tosca_template_dict
def handle_delivery(message):
logger.info("Got: " + str(message))
try:
message = message.decode()
except (UnicodeDecodeError, AttributeError):
pass
parsed_json_message = json.loads(message)
owner = parsed_json_message['owner']
tosca_file_name = 'tosca_template'
tosca_template_dict = parsed_json_message['toscaTemplate']
tosca_template_path = save_tosca_template(tosca_template_dict)
tosca_helper = ToscaHelper(sure_tosca_base_url, tosca_template_path)
# nodes_to_deploy = tosca_helper.get_application_nodes()
nodes = tosca_helper.get_deployment_node_pipeline()
deployService = DeployService(semaphore_base_url=semaphore_base_url,
semaphore_username=semaphore_username,
semaphore_password=semaphore_password,
vms=tosca_helper.get_vms())
try:
for node in nodes:
updated_node = deployService.deploy(node)
if isinstance(updated_node, list):
for node in updated_node:
tosca_template_dict = tosca_helper.set_node(
node, tosca_template_dict)
# logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
else:
tosca_template_dict = tosca_helper.set_node(
updated_node, tosca_template_dict)
# logger.info("tosca_template_dict :" + json.dumps(tosca_template_dict))
response = {'toscaTemplate': tosca_template_dict}
output_current_milli_time = int(round(time.time() * 1000))
response["creationDate"] = output_current_milli_time
logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response))
return json.dumps(response)
except Exception as e:
track = traceback.format_exc()
print(track)
raise
def test(self):
logger = logging.getLogger(__name__)
message_file_path = str(
Path.home()) + '/Downloads/message_deploy_request.json'
if os.path.isfile(message_file_path):
with open(message_file_path, 'r') as stream:
parsed_json_message = json.load(stream)
# owner = parsed_json_message['owner']
tosca_file_name = 'tosca_template'
tosca_template_dict = parsed_json_message['toscaTemplate']
tmp_path = tempfile.mkdtemp()
tosca_template_path = tmp_path + os.path.sep + 'toscaTemplate.yml'
with open(tosca_template_path, 'w') as outfile:
yaml.dump(tosca_template_dict,
outfile,
default_flow_style=False)
sure_tosca_base_url = 'http://127.0.0.1:8081/tosca-sure/1.0.0'
semaphore_base_url = 'http://127.0.0.1:3000/api'
tosca_service_is_up = ToscaHelper.service_is_up(
sure_tosca_base_url)
semaphore_is_up = ToscaHelper.service_is_up(semaphore_base_url)
if tosca_service_is_up and semaphore_is_up:
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
self.assertIsNotNone(tosca_helper.doc_id)
nodes_to_deploy = tosca_helper.get_application_nodes()
self.assertIsNotNone(nodes_to_deploy)
nodes_pairs = tosca_helper.get_deployment_node_pipeline()
self.assertIsNotNone(nodes_pairs)
username = '******'
deployService = DeployService(
polemarch_username=username,
polemarch_password='******',
semaphore_base_url=semaphore_base_url,
semaphore_username=username,
semaphore_password='******')
for node_pair in nodes_pairs:
deployService.deploy(node_pair)
def create_workflow_templates(self,
tosca_workflow_step=None,
organization_id=None,
node_templates=None,
step_name=None,
workflow_name=None):
awx_workflow_steps = {}
tosca_node = node_templates[tosca_workflow_step['target']]
activities = tosca_workflow_step['activities']
call_operation = None
for activity in activities:
if 'call_operation' in activity:
call_operation = activity['call_operation']
break
if 'interfaces' in tosca_node:
interfaces = tosca_node['interfaces']
interface_name = call_operation.split('.')[0]
tosca_interface_job = call_operation.split('.')[1]
ancestors = self.tosca_helper.get_interface_ancestors(
interface_name)
if 'tosca.interfaces.QC.Ansible' in ancestors:
workflow_template_node = {}
template = interfaces[interface_name][tosca_interface_job]
template_name = workflow_name + '.' + step_name
logger.info('Creating template: ' + template_name)
extra_variables = None
if not 'repository' in template['inputs']:
raise Exception('Workflow steps for: ' + template_name +
' have no repository: ' + str(template))
if 'inputs' in template and 'repository' in template['inputs']:
repository_url = template['inputs']['repository']
project_id = self.create_project(
project_name=repository_url,
scm_url=repository_url,
scm_branch='master',
scm_type='git',
organization_id=organization_id)
workflow_template_node[template_name] = {
'project': project_id[0]
}
if not 'inventory' in template['inputs']:
raise Exception(tosca_interface_job +
' has no inventory')
inventory = template['inputs']['inventory']
inventory_id = self.create_inventory(
inventory_name=template_name,
inventory=inventory,
organization_id=organization_id)
workflow_template_node[template_name][
'inventory'] = inventory_id
logger.info('Created inventory: ' + str(inventory_id) +
' for :' + template_name)
if 'implementation' in template:
if 'extra_variables' in template['inputs']:
extra_variables = self.get_variables(
extra_variables=template['inputs']
['extra_variables'])
workflow_template_node[template_name][
'implementation'] = template['implementation']
if not workflow_template_node[template_name]['inventory']:
raise Exception(template_name +
' is missing inventory')
credentials = ToscaHelper.extract_credentials_from_node(
tosca_node)
workflow_template_node[template_name][
'job_template'] = self.create_job_template(
workflow_template_node,
credentials=credentials,
organization_id=organization_id,
extra_vars=extra_variables)[0]
else:
raise Exception(template_name + ' has no implementation!')
if workflow_template_node:
logger.info('Created workflow_template_node: ' +
str(workflow_template_node))
awx_workflow_steps.update(workflow_template_node)
return awx_workflow_steps
def awx(tosca_template_path=None, tosca_template_dict=None):
awx_inst = None
global tosca_helper
try:
tosca_service_is_up = ToscaHelper.service_is_up(sure_tosca_base_url)
if tosca_service_is_up:
logger.info('Initializing ToscaHelper')
tosca_helper = ToscaHelper(sure_tosca_base_url,
tosca_template_path)
node_templates = tosca_template_dict['topology_template'][
'node_templates']
logger.info('Initializing AWXService')
awx_inst = AWXService(api_url=awx_base_url,
username=awx_username,
password=awx_password,
tosca_helper=tosca_helper)
logger.info('Creating organization: sdia')
organization_id = awx_inst.create_organization('sdia')
for tosca_node_name in node_templates:
tosca_node = node_templates[tosca_node_name]
logger.info('Resolving function values for: ' +
tosca_node_name)
tosca_node = tosca_helper.resolve_function_values(tosca_node)
workflows = tosca_helper.get_workflows()
if workflows:
for workflow_name in workflows:
topology_template_workflow_steps = {}
workflow = workflows[workflow_name]
can_run = tosca_helper.check_workflow_preconditions(
workflow, tosca_template_dict)
logger.info('workflow: ' + workflow_name + ' can run: ' +
str(can_run))
if can_run:
steps = workflow['steps']
for step_name in steps:
logger.info('Created step_name: ' + str(step_name))
node_workflow_steps = awx_inst.create_workflow_templates(
tosca_workflow_step=steps[step_name],
organization_id=organization_id,
node_templates=node_templates,
step_name=step_name,
workflow_name=workflow_name)
topology_template_workflow_steps.update(
node_workflow_steps)
tosca_template_dict = execute_workflows(
workflow=workflow,
workflow_name=workflow_name,
topology_template_workflow_steps=
topology_template_workflow_steps,
awx=awx_inst,
tosca_template_dict=tosca_template_dict)
else:
raise Exception('Could not connect to sure tosca at ' +
sure_tosca_base_url)
except Exception as ex:
track = traceback.format_exc()
print(track)
raise
finally:
if awx_inst and delete_templates_after_execution:
awx_inst.clean_up_execution()
tosca_template_dict = encrypt_credentials(tosca_template_dict)
response = {'toscaTemplate': tosca_template_dict}
logger.info("Returning Deployment")
logger.info("Output message:" + json.dumps(response))
return json.dumps(response)