class TestTask(unittest.TestCase):
"""Task unit test stubs"""
def setUp(self):
self.semaphore_base_url = 'http://localhost:3000/api'
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
self.username = '******'
self.password = '******'
self.project_name = 'test'
self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \
'-----END RSA PRIVATE KEY-----'
vms = []
vm = {}
vm['role'] = 'master'
vm['public_ip'] = '192.168.1.10'
vm['user_name'] = 'user_name'
vms.append(vm)
vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.11'
vm['role'] = 'worker'
vm['public_ip'] = '192.168.1.12'
vms.append(vm)
self.inventory_contents = yaml.dump(self.build_yml_inventory(vms),
default_flow_style=False)
self.git_url = 'https://github.com/skoulouzis/playbooks.git'
self.playbook_name = 'get_ip_addresses.yaml'
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url,
self.username,
self.password)
def tearDown(self):
pass
def testTask(self):
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
project_id = self.semaphore_helper.create_project(
self.project_name)
key_id = self.semaphore_helper.create_ssh_key(
self.project_name, project_id, self.private_key)
inventory_id = self.semaphore_helper.create_inventory(
self.project_name, project_id, key_id, self.inventory_contents)
repository_id = self.semaphore_helper.create_repository(
self.project_name, project_id, key_id, self.git_url)
template_id = self.semaphore_helper.create_template(
project_id, key_id, inventory_id, repository_id,
self.playbook_name)
task_id = self.semaphore_helper.execute_task(
project_id, template_id, self.playbook_name)
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
task_id = self.semaphore_helper.execute_task(
project_id, template_id, self.playbook_name)
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
task_id = self.semaphore_helper.execute_task(
project_id, template_id, 'mount.yaml')
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
def build_yml_inventory(self, vms):
inventory = {}
all = {}
vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0]['user_name']
children = {}
for vm in vms:
role = vm['role']
public_ip = vm['public_ip']
if role not in children:
hosts = {}
else:
hosts = children[role]
hosts[public_ip] = vars
children[role] = hosts
all['children'] = children
inventory['all'] = all
return inventory
class AnsibleService:
def __init__(self,
semaphore_base_url=None,
semaphore_username=None,
semaphore_password=None):
self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url,
self.semaphore_username,
self.semaphore_password)
self.repository_id = None
self.template_id = None
def execute(self, application, interface_type, vms, env_vars=None):
name = application.name
desired_state = None
tasks_outputs = {}
interfaces = application.node_template.interfaces
if 'current_state' in application.node_template.attributes:
current_state = application.node_template.attributes[
'current_state']
if 'desired_state' in application.node_template.attributes:
desired_state = application.node_template.attributes[
'desired_state']
if desired_state:
now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(
application.name + '_' + str(now))
inventory_dict = self.build_inventory(
vms, application_name=application.name)
inventory_contents = yaml.dump(inventory_dict,
default_flow_style=False)
private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(
application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(
application.name, project_id, key_id, inventory_contents)
if current_state and current_state != desired_state:
if 'RUNNING' == desired_state:
interface = interfaces[interface_type]
create = interface['create']
inputs = create['inputs']
git_url = inputs['repository']
task_id, tasks_outputs = self.create_node(
interfaces=interfaces,
interface_type=interface_type,
env_vars=env_vars,
project_id=project_id,
name=name,
application=application,
vms=vms,
key_id=key_id,
inventory_id=inventory_id)
tasks_outputs[
task_id] = self.semaphore_helper.get_task_outputs(
project_id, task_id)
# if 'resources' in inputs:
# playbook_names = inputs['resources']
# for playbook_name in playbook_names:
# environment_id = None
# if env_vars:
# environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
# arguments = None
# if application.name == 'gluster_fs' or application.name == 'glusterfs' or application.name == 'tic':
# for vm in vms:
# attributes = vm.node_template.attributes
# if attributes['role'] == 'master':
# master_ip = attributes['public_ip']
# break
# arguments = '["-u","vm_user","--extra-vars","gluster_cluster_host0=\'' + master_ip \
# + '\' gluster_cluster_volume=\'gfs0\' devmode=\'false\' ' \
# 'device_path=\'/dev/xvdh\' gfs_size=\'15G\'"]'
# task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
# environment_id=environment_id, arguments=arguments)
# count = 0
# while self.semaphore_helper.get_task(project_id, task_id).status != 'success':
# task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
# environment_id=environment_id, arguments=arguments)
# count += 1
# if count >= 3:
# msg = ' '
# for out in self.semaphore_helper.get_task_outputs(project_id, task_id):
# msg = msg + ' ' + out.output
# raise Exception(
# 'Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task(project_id,
# task_id).status + ' Output: ' + msg)
#
# tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
if 'configure' in interface and self.semaphore_helper.get_task(
project_id, task_id
).status == 'success' and 'resources' in inputs:
task_id = self.configure_node(
interface=interface,
project_id=project_id,
name=name,
env_vars=env_vars,
key_id=key_id,
inventory_id=inventory_id)
tasks_outputs[
task_id] = self.semaphore_helper.get_task_outputs(
project_id, task_id)
# configure = interface['configure']
# inputs = configure['inputs']
# git_url = inputs['repository']
# playbook_names = inputs['resources']
# for playbook_name in playbook_names:
# environment_id = None
# if env_vars:
# environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars)
# task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name,
# environment_id=environment_id)
# if self.semaphore_helper.get_task(project_id, task_id).status != 'success':
# msg = ''
# for out in self.semaphore_helper.get_task_outputs(project_id, task_id):
# msg = msg + out.output
# raise Exception(
# 'Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task(project_id,
# task_id).status + ' Output: ' + msg)
# # logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id))
# tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id)
return tasks_outputs
def build_inventory(self, vms, application_name=None):
# loader = DataLoader()
# inventory = InventoryManager(loader=loader)
# variable_manager = VariableManager()
vars = {}
vars['ansible_ssh_common_args'] = '"-o StrictHostKeyChecking=no"'
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
vars['ansible_python_interpreter'] = '/usr/bin/python3'
if application_name == 'gluster_fs' or application_name == 'glusterfs':
return self.build_glusterfs_inventory(vms, vars)
if application_name == 'tic':
return self.build_tic_inventory(vms, vars)
inventory = {}
all = {}
children = {}
for vm in vms:
attributes = vm.node_template.attributes
roles = []
roles.append(attributes['role'])
public_ip = attributes['public_ip']
for role in roles:
if role not in children:
hosts = {}
else:
hosts = children[role]
host = {}
host[public_ip] = vars
if 'hosts' in hosts:
hosts['hosts'][public_ip] = vars
else:
hosts['hosts'] = host
children[role] = hosts
all['children'] = children
inventory['all'] = all
return inventory
def get_private_key(self, vms):
private_key = vms[0].node_template.attributes['user_key_pair']['keys'][
'private_key']
return base64.b64decode(private_key).decode('utf-8').replace(
r'\n', '\n')
def run_task(self,
name,
project_id,
key_id,
git_url,
inventory_id,
playbook_name,
environment_id=None,
arguments=None):
logger.info('project_id: ' + str(project_id) + ' task name: ' +
str(name) + ' git url: ' + git_url + ' playbook: ' +
playbook_name)
self.repository_id = self.semaphore_helper.create_repository(
name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(
project_id, key_id, inventory_id, self.repository_id,
playbook_name, arguments)
task_id = self.semaphore_helper.execute_task(
project_id,
template_id,
playbook_name,
environment_id=environment_id)
task = self.semaphore_helper.get_task(project_id, task_id)
last_status = ''
while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id)
this_status = task.status
if last_status != this_status:
logger.info('task name: ' + name + ', task status: ' +
str(task.status))
last_status = this_status
sleep(6)
return task_id
def build_glusterfs_inventory(self, vms, vars):
inventory = {}
all = {}
children = {}
gfs_count = 1
for vm in vms:
attributes = vm.node_template.attributes
roles = []
roles.append('gfscluster')
public_ip = attributes['public_ip']
vm_vars = {'ansible_host': public_ip}
vm_vars.update(vars)
for role in roles:
if role not in children:
hosts = {}
else:
hosts = children[role]
if 'hosts' in hosts:
hosts['hosts']['gfs' + str(gfs_count)] = vm_vars
else:
host = {}
host['gfs' + str(gfs_count)] = vm_vars
hosts['hosts'] = host
gfs_count += 1
children[role] = hosts
all['children'] = children
inventory['all'] = all
return inventory
def build_tic_inventory(self, vms, vars):
inventory = {}
all = {}
children = {}
for vm in vms:
attributes = vm.node_template.attributes
public_ip = attributes['public_ip']
vm_vars = {'ansible_host': public_ip}
vm_vars['ansible_ssh_common_args'] = '-o StrictHostKeyChecking=no'
vm_vars.update(vars)
roles = []
if attributes['role'] == 'master':
roles.append('swarm_manager_prime')
roles.append('swarm_managers')
elif attributes['role'] == 'worker':
roles.append('swarm_workers')
fabric_count = 0
for role in roles:
if role not in children:
hosts = {}
else:
hosts = children[role]
if 'hosts' in hosts:
if role == 'swarm_manager_prime' or role == 'swarm_managers':
host = {'hlf' + str(fabric_count): vm_vars}
else:
fabric_count += 1
host = {'hlf' + str(fabric_count): vm_vars}
hosts['hosts'] = host
else:
if role == 'swarm_manager_prime' or role == 'swarm_managers':
host = {'hlf' + str(fabric_count): vm_vars}
else:
fabric_count += 1
host = {'hlf' + str(fabric_count): vm_vars}
hosts['hosts'] = host
children[role] = hosts
all['children'] = children
inventory['all'] = all
return inventory
def create_node(self,
interfaces=None,
interface_type=None,
env_vars=None,
project_id=None,
name=None,
application=None,
vms=None,
key_id=None,
inventory_id=None):
interface = interfaces[interface_type]
create = interface['create']
inputs = create['inputs']
git_url = inputs['repository']
if 'resources' in inputs:
playbook_names = inputs['resources']
for playbook_name in playbook_names:
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(
project_id, name, env_vars)
arguments = None
if application.name == 'gluster_fs' or application.name == 'glusterfs' or application.name == 'tic':
for vm in vms:
attributes = vm.node_template.attributes
if attributes['role'] == 'master':
master_ip = attributes['public_ip']
break
arguments = '["-u","vm_user","--extra-vars","gluster_cluster_host0=\'' + master_ip \
+ '\' gluster_cluster_volume=\'gfs0\' devmode=\'false\' ' \
'device_path=\'/dev/xvdh\' gfs_size=\'15G\'"]'
task_id = self.run_task(name,
project_id,
key_id,
git_url,
inventory_id,
playbook_name,
environment_id=environment_id,
arguments=arguments)
count = 0
while self.semaphore_helper.get_task(
project_id, task_id).status != 'success':
task_id = self.run_task(name,
project_id,
key_id,
git_url,
inventory_id,
playbook_name,
environment_id=environment_id,
arguments=arguments)
count += 1
if count >= 3:
msg = ' '
for out in self.semaphore_helper.get_task_outputs(
project_id, task_id):
msg = msg + ' ' + out.output
raise Exception('Task: ' + playbook_name +
' failed. ' +
self.semaphore_helper.get_task(
project_id, task_id).status +
' Output: ' + msg)
return task_id
def configure_node(self,
interface=None,
project_id=None,
name=None,
env_vars=None,
key_id=None,
inventory_id=None):
configure = interface['configure']
inputs = configure['inputs']
git_url = inputs['repository']
playbook_names = inputs['resources']
for playbook_name in playbook_names:
environment_id = None
if env_vars:
environment_id = self.semaphore_helper.create_environment(
project_id, name, env_vars)
task_id = self.run_task(name,
project_id,
key_id,
git_url,
inventory_id,
playbook_name,
environment_id=environment_id)
if self.semaphore_helper.get_task(project_id,
task_id).status != 'success':
msg = ''
for out in self.semaphore_helper.get_task_outputs(
project_id, task_id):
msg = msg + out.output
raise Exception('Task: ' + playbook_name + ' failed. ' +
self.semaphore_helper.get_task(
project_id, task_id).status + ' Output: ' +
msg)
return task_id
class AnsibleService:
def __init__(self,
semaphore_base_url=None,
semaphore_username=None,
semaphore_password=None):
self.semaphore_base_url = semaphore_base_url
self.semaphore_username = semaphore_username
self.semaphore_password = semaphore_password
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url,
self.semaphore_username,
self.semaphore_password)
self.repository_id = None
self.template_id = None
def execute(self, nodes_pair):
vms = nodes_pair[0]
application = nodes_pair[1]
name = application.name
desired_state = None
interfaces = application.node_template.interfaces
if 'current_state' in application.node_template.attributes:
current_state = application.node_template.attributes[
'current_state']
if 'desired_state' in application.node_template.attributes:
desired_state = application.node_template.attributes[
'desired_state']
if desired_state:
now = datetime.datetime.now()
project_id = self.semaphore_helper.create_project(
application.name + '_' + str(now))
inventory_contents = yaml.dump(self.build_yml_inventory(vms),
default_flow_style=False)
private_key = self.get_private_key(vms)
key_id = self.semaphore_helper.create_ssh_key(
application.name, project_id, private_key)
inventory_id = self.semaphore_helper.create_inventory(
application.name, project_id, key_id, inventory_contents)
if 'RUNNING' == desired_state:
standard = interfaces['Standard']
create = standard['create']
inputs = create['inputs']
git_url = inputs['repository']
playbook_names = inputs['resources']
for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id, git_url,
inventory_id, playbook_name)
if self.semaphore_helper.get_task(
project_id, task_id).status != 'success':
break
if self.semaphore_helper.get_task(project_id,
task_id).status == 'success':
configure = standard['configure']
inputs = configure['inputs']
git_url = inputs['repository']
playbook_names = inputs['resources']
for playbook_name in playbook_names:
task_id = self.run_task(name, project_id, key_id,
git_url, inventory_id,
playbook_name)
if self.semaphore_helper.get_task(
project_id, task_id).status != 'success':
break
def build_yml_inventory(self, vms):
# loader = DataLoader()
# inventory = InventoryManager(loader=loader)
# variable_manager = VariableManager()
inventory = {}
all = {}
vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name']
children = {}
for vm in vms:
attributes = vm.node_template.attributes
role = attributes['role']
public_ip = attributes['public_ip']
if role not in children:
hosts = {}
else:
hosts = children[role]
host = {}
host[public_ip] = vars
hosts['hosts'] = host
children[role] = hosts
# inventory.add_group(role)
# inventory.add_host(public_ip,group=role)
all['children'] = children
inventory['all'] = all
return inventory
def get_private_key(self, vms):
private_key = vms[0].node_template.attributes['user_key_pair']['keys'][
'private_key']
return base64.b64decode(private_key).decode('utf-8').replace(
r'\n', '\n')
def run_task(self, name, project_id, key_id, git_url, inventory_id,
playbook_name):
logger.info('project_id: ' + str(project_id) + ' task name: ' +
str(name) + ' git url: ' + git_url + ' playbook: ' +
playbook_name)
self.repository_id = self.semaphore_helper.create_repository(
name, project_id, key_id, git_url)
template_id = self.semaphore_helper.create_template(
project_id, key_id, inventory_id, self.repository_id,
playbook_name)
task_id = self.semaphore_helper.execute_task(project_id, template_id,
playbook_name)
task = self.semaphore_helper.get_task(project_id, task_id)
last_output = ''
while task.status == 'waiting' or task.status == 'running':
task = self.semaphore_helper.get_task(project_id, task_id)
logger.info('task name: ' + name + ' task status: ' +
str(task.status))
task_outputs = self.semaphore_helper.get_task_outputs(
project_id, task_id)
this_output = task_outputs[len(task_outputs) - 1].output.replace(
r'\n', '\n').replace(r'\r', '\r')
if last_output != this_output:
logger.info('task output: ' + str(this_output))
last_output = this_output
# logger.info('task output: ' + str(latask name:st_output))
sleep(3)
return task_id
class TestTask(unittest.TestCase):
"""Task unit test stubs"""
def setUp(self):
self.semaphore_base_url = 'http://127.0.0.1:3000/api'
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
self.username = '******'
self.password = '******'
now = datetime.datetime.now()
self.project_name = 'project_name_test' + '_' + str(now)
self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \
'-----END RSA PRIVATE KEY-----'
vms = []
vm = {}
vm['role'] = 'master'
vm['public_ip'] = '127.0.0.1'
vm['semaphore'] = 'semaphore'
vms.append(vm)
vm['role'] = 'worker'
vm['public_ip'] = '127.0.0.1'
vm['role'] = 'worker'
vm['public_ip'] = '127.0.0.1'
vms.append(vm)
self.inventory_contents = yaml.dump(self.build_yml_inventory(vms),
default_flow_style=False)
self.git_url = 'https://github.com/QCDIS/playbooks.git'
self.playbook_name = 'get_ip_addresses.yaml'
self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url,
self.username,
self.password)
def tearDown(self):
pass
def testTask(self):
if SemaphoreHelper.service_is_up(self.semaphore_base_url):
project_id = self.semaphore_helper.create_project(
self.project_name)
now = datetime.datetime.now()
key_name = 'key_name' + str(now)
key_id = self.semaphore_helper.create_ssh_key(
key_name, project_id, self.private_key)
now = datetime.datetime.now()
inventory_name = 'inventory_name' + str(now)
inventory_id = self.semaphore_helper.create_inventory(
inventory_name, project_id, key_id, self.inventory_contents)
now = datetime.datetime.now()
repository_name = 'repository_name' + str(now)
repository_id = self.semaphore_helper.create_repository(
repository_name, project_id, key_id, self.git_url)
template_id = self.semaphore_helper.create_template(
project_id, key_id, inventory_id, repository_id,
self.playbook_name)
env_vars = {"SOME_VAR": "the contents"}
now = datetime.datetime.now()
environment_name = 'environment_name_name_' + str(now)
environment_id = self.semaphore_helper.create_environment(
project_id, environment_name, env_vars)
task_id = self.semaphore_helper.execute_task(
project_id,
template_id,
self.playbook_name,
environment_id=environment_id)
for x in range(0, 2):
task = self.semaphore_helper.get_task(project_id, task_id)
print(task)
# task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# print(task_output)
sleep(1)
# task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name)
# for x in range(0, 2):
# task = self.semaphore_helper.get_task(project_id, task_id)
# print(task)
# # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# # print(task_output)
# sleep(1)
# task_id = self.semaphore_helper.execute_task(project_id, template_id, 'get_ip_addresses.yaml')
# for x in range(0, 2):
# task = self.semaphore_helper.get_task(project_id, task_id)
# print(task)
# # task_output = self.semaphore_helper.get_task_output(project_id, task_id)
# # print(task_output)
# sleep(1)
def build_yml_inventory(self, vms):
inventory = {}
all = {}
vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'}
vars['ansible_ssh_user'] = vms[0]['semaphore']
children = {}
for vm in vms:
role = vm['role']
public_ip = vm['public_ip']
if role not in children:
hosts = {}
else:
hosts = children[role]
hosts[public_ip] = vars
children[role] = hosts
all['children'] = children
inventory['all'] = all
return inventory