class AnsibleService: def __init__(self, semaphore_base_url=None, semaphore_username=None, semaphore_password=None): self.semaphore_base_url = semaphore_base_url self.semaphore_username = semaphore_username self.semaphore_password = semaphore_password self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.semaphore_username, self.semaphore_password) self.repository_id = None self.template_id = None def execute(self, application, interface_type, vms, env_vars=None): name = application.name desired_state = None tasks_outputs = {} interfaces = application.node_template.interfaces if 'current_state' in application.node_template.attributes: current_state = application.node_template.attributes[ 'current_state'] if 'desired_state' in application.node_template.attributes: desired_state = application.node_template.attributes[ 'desired_state'] if desired_state: now = datetime.datetime.now() project_id = self.semaphore_helper.create_project( application.name + '_' + str(now)) inventory_dict = self.build_inventory( vms, application_name=application.name) inventory_contents = yaml.dump(inventory_dict, default_flow_style=False) private_key = self.get_private_key(vms) key_id = self.semaphore_helper.create_ssh_key( application.name, project_id, private_key) inventory_id = self.semaphore_helper.create_inventory( application.name, project_id, key_id, inventory_contents) if current_state and current_state != desired_state: if 'RUNNING' == desired_state: interface = interfaces[interface_type] create = interface['create'] inputs = create['inputs'] git_url = inputs['repository'] task_id, tasks_outputs = self.create_node( interfaces=interfaces, interface_type=interface_type, env_vars=env_vars, project_id=project_id, name=name, application=application, vms=vms, key_id=key_id, inventory_id=inventory_id) tasks_outputs[ task_id] = self.semaphore_helper.get_task_outputs( project_id, task_id) # if 'resources' in inputs: # playbook_names = inputs['resources'] # for playbook_name in playbook_names: # environment_id = None # if env_vars: # environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars) # arguments = None # if application.name == 'gluster_fs' or application.name == 'glusterfs' or application.name == 'tic': # for vm in vms: # attributes = vm.node_template.attributes # if attributes['role'] == 'master': # master_ip = attributes['public_ip'] # break # arguments = '["-u","vm_user","--extra-vars","gluster_cluster_host0=\'' + master_ip \ # + '\' gluster_cluster_volume=\'gfs0\' devmode=\'false\' ' \ # 'device_path=\'/dev/xvdh\' gfs_size=\'15G\'"]' # task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, # environment_id=environment_id, arguments=arguments) # count = 0 # while self.semaphore_helper.get_task(project_id, task_id).status != 'success': # task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, # environment_id=environment_id, arguments=arguments) # count += 1 # if count >= 3: # msg = ' ' # for out in self.semaphore_helper.get_task_outputs(project_id, task_id): # msg = msg + ' ' + out.output # raise Exception( # 'Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task(project_id, # task_id).status + ' Output: ' + msg) # # tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id) if 'configure' in interface and self.semaphore_helper.get_task( project_id, task_id ).status == 'success' and 'resources' in inputs: task_id = self.configure_node( interface=interface, project_id=project_id, name=name, env_vars=env_vars, key_id=key_id, inventory_id=inventory_id) tasks_outputs[ task_id] = self.semaphore_helper.get_task_outputs( project_id, task_id) # configure = interface['configure'] # inputs = configure['inputs'] # git_url = inputs['repository'] # playbook_names = inputs['resources'] # for playbook_name in playbook_names: # environment_id = None # if env_vars: # environment_id = self.semaphore_helper.create_environment(project_id, name, env_vars) # task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, # environment_id=environment_id) # if self.semaphore_helper.get_task(project_id, task_id).status != 'success': # msg = '' # for out in self.semaphore_helper.get_task_outputs(project_id, task_id): # msg = msg + out.output # raise Exception( # 'Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task(project_id, # task_id).status + ' Output: ' + msg) # # logger.info('playbook: ' + playbook_name + ' task_id: ' + str(task_id)) # tasks_outputs[task_id] = self.semaphore_helper.get_task_outputs(project_id, task_id) return tasks_outputs def build_inventory(self, vms, application_name=None): # loader = DataLoader() # inventory = InventoryManager(loader=loader) # variable_manager = VariableManager() vars = {} vars['ansible_ssh_common_args'] = '"-o StrictHostKeyChecking=no"' vars['ansible_ssh_user'] = vms[0].node_template.properties['user_name'] vars['ansible_python_interpreter'] = '/usr/bin/python3' if application_name == 'gluster_fs' or application_name == 'glusterfs': return self.build_glusterfs_inventory(vms, vars) if application_name == 'tic': return self.build_tic_inventory(vms, vars) inventory = {} all = {} children = {} for vm in vms: attributes = vm.node_template.attributes roles = [] roles.append(attributes['role']) public_ip = attributes['public_ip'] for role in roles: if role not in children: hosts = {} else: hosts = children[role] host = {} host[public_ip] = vars if 'hosts' in hosts: hosts['hosts'][public_ip] = vars else: hosts['hosts'] = host children[role] = hosts all['children'] = children inventory['all'] = all return inventory def get_private_key(self, vms): private_key = vms[0].node_template.attributes['user_key_pair']['keys'][ 'private_key'] return base64.b64decode(private_key).decode('utf-8').replace( r'\n', '\n') def run_task(self, name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=None, arguments=None): logger.info('project_id: ' + str(project_id) + ' task name: ' + str(name) + ' git url: ' + git_url + ' playbook: ' + playbook_name) self.repository_id = self.semaphore_helper.create_repository( name, project_id, key_id, git_url) template_id = self.semaphore_helper.create_template( project_id, key_id, inventory_id, self.repository_id, playbook_name, arguments) task_id = self.semaphore_helper.execute_task( project_id, template_id, playbook_name, environment_id=environment_id) task = self.semaphore_helper.get_task(project_id, task_id) last_status = '' while task.status == 'waiting' or task.status == 'running': task = self.semaphore_helper.get_task(project_id, task_id) this_status = task.status if last_status != this_status: logger.info('task name: ' + name + ', task status: ' + str(task.status)) last_status = this_status sleep(6) return task_id def build_glusterfs_inventory(self, vms, vars): inventory = {} all = {} children = {} gfs_count = 1 for vm in vms: attributes = vm.node_template.attributes roles = [] roles.append('gfscluster') public_ip = attributes['public_ip'] vm_vars = {'ansible_host': public_ip} vm_vars.update(vars) for role in roles: if role not in children: hosts = {} else: hosts = children[role] if 'hosts' in hosts: hosts['hosts']['gfs' + str(gfs_count)] = vm_vars else: host = {} host['gfs' + str(gfs_count)] = vm_vars hosts['hosts'] = host gfs_count += 1 children[role] = hosts all['children'] = children inventory['all'] = all return inventory def build_tic_inventory(self, vms, vars): inventory = {} all = {} children = {} for vm in vms: attributes = vm.node_template.attributes public_ip = attributes['public_ip'] vm_vars = {'ansible_host': public_ip} vm_vars['ansible_ssh_common_args'] = '-o StrictHostKeyChecking=no' vm_vars.update(vars) roles = [] if attributes['role'] == 'master': roles.append('swarm_manager_prime') roles.append('swarm_managers') elif attributes['role'] == 'worker': roles.append('swarm_workers') fabric_count = 0 for role in roles: if role not in children: hosts = {} else: hosts = children[role] if 'hosts' in hosts: if role == 'swarm_manager_prime' or role == 'swarm_managers': host = {'hlf' + str(fabric_count): vm_vars} else: fabric_count += 1 host = {'hlf' + str(fabric_count): vm_vars} hosts['hosts'] = host else: if role == 'swarm_manager_prime' or role == 'swarm_managers': host = {'hlf' + str(fabric_count): vm_vars} else: fabric_count += 1 host = {'hlf' + str(fabric_count): vm_vars} hosts['hosts'] = host children[role] = hosts all['children'] = children inventory['all'] = all return inventory def create_node(self, interfaces=None, interface_type=None, env_vars=None, project_id=None, name=None, application=None, vms=None, key_id=None, inventory_id=None): interface = interfaces[interface_type] create = interface['create'] inputs = create['inputs'] git_url = inputs['repository'] if 'resources' in inputs: playbook_names = inputs['resources'] for playbook_name in playbook_names: environment_id = None if env_vars: environment_id = self.semaphore_helper.create_environment( project_id, name, env_vars) arguments = None if application.name == 'gluster_fs' or application.name == 'glusterfs' or application.name == 'tic': for vm in vms: attributes = vm.node_template.attributes if attributes['role'] == 'master': master_ip = attributes['public_ip'] break arguments = '["-u","vm_user","--extra-vars","gluster_cluster_host0=\'' + master_ip \ + '\' gluster_cluster_volume=\'gfs0\' devmode=\'false\' ' \ 'device_path=\'/dev/xvdh\' gfs_size=\'15G\'"]' task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=environment_id, arguments=arguments) count = 0 while self.semaphore_helper.get_task( project_id, task_id).status != 'success': task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=environment_id, arguments=arguments) count += 1 if count >= 3: msg = ' ' for out in self.semaphore_helper.get_task_outputs( project_id, task_id): msg = msg + ' ' + out.output raise Exception('Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task( project_id, task_id).status + ' Output: ' + msg) return task_id def configure_node(self, interface=None, project_id=None, name=None, env_vars=None, key_id=None, inventory_id=None): configure = interface['configure'] inputs = configure['inputs'] git_url = inputs['repository'] playbook_names = inputs['resources'] for playbook_name in playbook_names: environment_id = None if env_vars: environment_id = self.semaphore_helper.create_environment( project_id, name, env_vars) task_id = self.run_task(name, project_id, key_id, git_url, inventory_id, playbook_name, environment_id=environment_id) if self.semaphore_helper.get_task(project_id, task_id).status != 'success': msg = '' for out in self.semaphore_helper.get_task_outputs( project_id, task_id): msg = msg + out.output raise Exception('Task: ' + playbook_name + ' failed. ' + self.semaphore_helper.get_task( project_id, task_id).status + ' Output: ' + msg) return task_id
class TestTask(unittest.TestCase): """Task unit test stubs""" def setUp(self): self.semaphore_base_url = 'http://127.0.0.1:3000/api' if SemaphoreHelper.service_is_up(self.semaphore_base_url): self.username = '******' self.password = '******' now = datetime.datetime.now() self.project_name = 'project_name_test' + '_' + str(now) self.private_key = '-----BEGIN RSA PRIVATE KEY-----MIIEowIBAAKCAQEAg0blRNV6cm3RTiivpzE8HR4JzKZRVIBZ7bxeNoMz0' \ '-----END RSA PRIVATE KEY-----' vms = [] vm = {} vm['role'] = 'master' vm['public_ip'] = '127.0.0.1' vm['semaphore'] = 'semaphore' vms.append(vm) vm['role'] = 'worker' vm['public_ip'] = '127.0.0.1' vm['role'] = 'worker' vm['public_ip'] = '127.0.0.1' vms.append(vm) self.inventory_contents = yaml.dump(self.build_yml_inventory(vms), default_flow_style=False) self.git_url = 'https://github.com/QCDIS/playbooks.git' self.playbook_name = 'get_ip_addresses.yaml' self.semaphore_helper = SemaphoreHelper(self.semaphore_base_url, self.username, self.password) def tearDown(self): pass def testTask(self): if SemaphoreHelper.service_is_up(self.semaphore_base_url): project_id = self.semaphore_helper.create_project( self.project_name) now = datetime.datetime.now() key_name = 'key_name' + str(now) key_id = self.semaphore_helper.create_ssh_key( key_name, project_id, self.private_key) now = datetime.datetime.now() inventory_name = 'inventory_name' + str(now) inventory_id = self.semaphore_helper.create_inventory( inventory_name, project_id, key_id, self.inventory_contents) now = datetime.datetime.now() repository_name = 'repository_name' + str(now) repository_id = self.semaphore_helper.create_repository( repository_name, project_id, key_id, self.git_url) template_id = self.semaphore_helper.create_template( project_id, key_id, inventory_id, repository_id, self.playbook_name) env_vars = {"SOME_VAR": "the contents"} now = datetime.datetime.now() environment_name = 'environment_name_name_' + str(now) environment_id = self.semaphore_helper.create_environment( project_id, environment_name, env_vars) task_id = self.semaphore_helper.execute_task( project_id, template_id, self.playbook_name, environment_id=environment_id) for x in range(0, 2): task = self.semaphore_helper.get_task(project_id, task_id) print(task) # task_output = self.semaphore_helper.get_task_output(project_id, task_id) # print(task_output) sleep(1) # task_id = self.semaphore_helper.execute_task(project_id, template_id, self.playbook_name) # for x in range(0, 2): # task = self.semaphore_helper.get_task(project_id, task_id) # print(task) # # task_output = self.semaphore_helper.get_task_output(project_id, task_id) # # print(task_output) # sleep(1) # task_id = self.semaphore_helper.execute_task(project_id, template_id, 'get_ip_addresses.yaml') # for x in range(0, 2): # task = self.semaphore_helper.get_task(project_id, task_id) # print(task) # # task_output = self.semaphore_helper.get_task_output(project_id, task_id) # # print(task_output) # sleep(1) def build_yml_inventory(self, vms): inventory = {} all = {} vars = {'ansible_ssh_common_args': '-o StrictHostKeyChecking=no'} vars['ansible_ssh_user'] = vms[0]['semaphore'] children = {} for vm in vms: role = vm['role'] public_ip = vm['public_ip'] if role not in children: hosts = {} else: hosts = children[role] hosts[public_ip] = vars children[role] = hosts all['children'] = children inventory['all'] = all return inventory