def map_sync_inputs(self, controller_user): slog.debug("controller user: %r" % controller_user) if not controller_user.controller.admin_user: slog.info("controller %r has no admin_user, skipping" % controller_user.controller) return # All users will have at least the 'user' role at their home site/tenant. # We must also check if the user should have the admin role roles = ['user'] # setup user home site roles at controller if not controller_user.user.site: raise Exception('Siteless user %s' % controller_user.user.email) else: swarm_manager_url = controller_user.controller.auth_url slog.info("swarm_manager_url: %s" % swarm_manager_url) (swarm_manager_address, docker_registry_port) = swarm_manager_url.split(':') slog.info("swarm_manager_address: %s docker_registry_port: %s" % (swarm_manager_address, docker_registry_port)) user_fields = { 'swarm_manager_address': swarm_manager_address, 'admin_user': controller_user.controller.admin_user, 'ansible_tag': '%s@%s' % (controller_user.user.email.replace('@', '-at-'), controller_user.controller.name) # name of ansible playbook } slog.info("user_fields: %s" % user_fields) return user_fields
def chk_svc_exist(self, instance, swarm_manager_address): duplicated_flag = False try: instance_name = '%s-%s-%d' % (instance.slice.service.name, instance.slice.name, instance.id) slog.debug("Service name to chkeck: %s" % instance_name) import docker docker_api_base_url = "tcp://%s:4243" % swarm_manager_address slog.debug("Docker API Base URL: %s" % docker_api_base_url) my_client = docker.DockerClient(base_url=docker_api_base_url) swarm_svc = my_client.services.get(instance_name) slog.debug("swarm_svc.id : %s" % swarm_svc.id) slog.debug("swarm_svc.name : %s" % swarm_svc.name) slog.debug("swarm_svc.attrs : %s" % swarm_svc.attrs) except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc())) slog.info( "There is no duplicated service name, I would create new service (%s)" % instance_name) duplicated_flag = False else: slog.info("There is duplicated service name (%s)" % instance_name) duplicated_flag = True return duplicated_flag
def map_sync_outputs(self, controller_image, res): slog.debug("Ansible playbook result: %s" % str(res)) slog.debug("Ansible playbook result[4]['image']['Id']: %s" % str(res[4]['image']['Id'])) controller_image.glance_image_id = str(res[4]['image']['Id']) if len(controller_image.glance_image_id) > 2: controller_image.backend_status = '1 - OK' else: controller_image.backend_status = '2 - Ansible playbook failure' controller_image.save()
def alloc_subnet(self, uuid): # 16 bits only slog.debug("uuid: %s" % str(uuid)) uuid_masked = uuid & 0xffff a = 10 b = uuid_masked >> 8 c = uuid_masked & 0xff d = 0 cidr = '%d.%d.%d.%d/24' % (a, b, c, d) slog.debug("cidr: %s" % str(cidr)) return cidr
def map_delete_inputs(self, controller_image): try: controller_register = json.loads( instance.node.site_deployment.controller.backend_register) slog.debug("controller_register: %s" % controller_register) if (controller_register.get('disabled', False)): slog.info('Controller %s is disabled' % instance.node.site_deployment.controller.name) raise InnocuousException( 'Controller %s is disabled' % instance.node.site_deployment.controller.name) swarm_manager_url = controller_image.controller.auth_url (swarm_manager_address, docker_registry_port) = swarm_manager_url.split(':') slog.info("swarm_manager_address: %s docker_registry_port: %s" % (swarm_manager_address, docker_registry_port)) input_fields = { 'swarm_manager_address': swarm_manager_address, 'docker_registry_port': docker_registry_port, 'image_file_path': controller_image.image.path, 'image_dir': os.path.dirname(controller_image.image.path), 'image_name': controller_image.image.name, 'image_tag': controller_image.image.tag, 'delete': True, 'ansible_tag': '%s@%s' % (controller_image.image.name, controller_image.controller.name ) # name of ansible playbook } slog.info("input_fields: %s" % str(input_fields)) return input_fields except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc()))
def map_sync_outputs(self, controller_network, res): slog.debug("ansible playbook ressult: %s" % str(res)) slog.debug("ansible playbook ressult[1][stdout]: %s" % str(res[1]['stdout'])) res_stdout = res[1]['stdout'] json_content = json.loads(res_stdout) slog.debug("json_content: %s" % str(json_content)) network_id = json_content[0]['Id'] slog.debug("network_id: %s" % str(network_id)) subnet_id = "%s-subnet-%s" % (network_id, self.cidr) slog.debug("subnet_id: %s" % str(subnet_id)) controller_network.net_id = network_id controller_network.subnet = self.cidr controller_network.subnet_id = subnet_id controller_network.backend_status = '1 - OK' controller_network.save()
def update_instance_with_ssh(self, inst_id): try: import subprocess import shlex new_inst = Instance.objects.get(id=inst_id) controller = new_inst.node.site_deployment.controller swarm_manager_url = controller.auth_url (swarm_manager_address, docker_registry_port) = swarm_manager_url.split(':') slog.info("swarm_manager_address: %s docker_registry_port: %s" % (swarm_manager_address, docker_registry_port)) volume_mount_opt = "--mount-add type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % ( new_inst.id, new_inst.volumes) ## DOCKER SERVICE UPDATE ssh_cmd = "ssh root@%s docker service update --force %s %s" % ( swarm_manager_address, volume_mount_opt, new_inst.instance_name) slog.debug("SSH CMD: %s" % ssh_cmd) popen = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdoutdata, stderrdata) = popen.communicate() slog.debug("popen result: %s etc data: %s" % (stdoutdata, stderrdata)) ## DOCKER SERVICE STATUS ssh_cmd = "ssh root@%s docker service ps %s" % ( swarm_manager_address, new_inst.instance_name) slog.debug("SSH CMD: %s" % ssh_cmd) popen = subprocess.Popen(ssh_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) (stdoutdata, stderrdata) = popen.communicate() slog.debug("popen result: %s etc data: %s" % (stdoutdata, stderrdata)) except Exception as ex: slog.info("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.info("%s" % str(traceback.format_exc())) return 0
def chk_net_exist(self, controller_network, swarm_manager_address): duplicated_flag = False try: slog.debug("network_name to check: %s" % controller_network.network.name) import docker docker_api_base_url = "tcp://%s:4243" % swarm_manager_address slog.info("Docker API Base URL: %s" % docker_api_base_url) my_client = docker.DockerClient(base_url=docker_api_base_url) net = my_client.networks.get(controller_network.network.name) slog.debug("network.id : %s" % net.id) slog.debug("network.name : %s" % net.name) slog.debug("network.attrs[IPAM][Config] : %s" % net.attrs['IPAM']['Config']) slog.debug("network.attrs : %s" % net.attrs) except Exception as ex: slog.info("Exception: %s" % str(ex.args)) slog.info("Exception: %s" % str(ex)) duplicated_flag = False # There is no same network. else: duplicated_flag = True # There is same network. slog.debug("network duplicated_flag: %s" % duplicated_flag) return duplicated_flag
def map_sync_inputs(self, controller_network): # make sure to not sync a shared network slog.debug("controller_network: %s" % str(controller_network)) slog.debug("controller_network.subnet: %s" % controller_network.subnet) slog.debug("controller_network.network_id: %s" % controller_network.network_id) slog.debug("controller_network.network.name: %s" % controller_network.network.name) slog.debug( "controller_network.network.template.shared_network_name: %s" % controller_network.network.template.shared_network_name) slog.debug( "controller_network.network.template.shared_network_id : %s" % controller_network.network.template.shared_network_id) # If shared_network_name is kuryr/libnetwork2:latest --> then, discard following if-statement. """ if (controller_network.network.template.shared_network_name or controller_network.network.template.shared_network_id): if controller_network.network.template.shared_network_name.__contains__("kuryr"): slog.debug("This is kuryr network") else: slog.debug("SyncStep.SYNC_WITHOUT_RUNNING: %s" % str(SyncStep.SYNC_WITHOUT_RUNNING)) return SyncStep.SYNC_WITHOUT_RUNNING """ slog.debug("network owner: %s creator: %s" % (controller_network.network.owner, controller_network.network.owner.creator)) if controller_network.network.owner and controller_network.network.owner.creator: return self.save_controller_network(controller_network) else: slog.info("I could not create network controller information %s" % controller_network) raise Exception('I could not create network controller %s' % controller_network)
def handle_update(self, network): slog.debug("network.name: %s id: %s" % (network.name, network.id)) expected_controllers = Controller.objects.all() slog.debug("(network.name: %s) Controllers List: %s" % (network.name, str(expected_controllers))) existing_controllers = [] for cn in ControllerNetwork.objects.all(): slog.debug( "(network.name: %s) controllernetwork.id: %s name: %s" % (network.name, cn.id, cn.subnet)) slog.debug("(network.name: %s) controllernetwork.network.id: %s" % (network.name, cn.network.id)) if cn.network.id == network.id: existing_controllers.append(cn.controller) existing_controller_ids = [c.id for c in existing_controllers] slog.debug("(network.name: %s) existing_controller_id list: %s" % (network.name, str(existing_controller_ids))) for expected_controller in expected_controllers: slog.debug("(network.name: %s) expected_controller: %s" % (network.name, str(expected_controller))) if expected_controller.id not in existing_controller_ids: slog.debug("(network.name: %s) expected_controller.id: %s" % (network.name, expected_controller.id)) lazy_blocked = False # check and see if some instance already exists for networkslice in network.networkslices.all(): slog.debug("(network.name: %s) networkslice: %s" % (network.name, str(networkslice))) found = False for instance in networkslice.slice.instances.all(): slog.debug( "(network.name: %s) instance.instance_name: %s" % (network.name, instance_name)) slog.debug( "(network.name: %s) instance.controller.id: %s expected_controller.id: %s" % (network.name, instance.node.site_deployment.controller.id, expected_controller.id)) if instance.node.site_deployment.controller.id == expected_controller.id: found = True slog.debug("(network.name: %s) found: %s" % (network.name, found)) if found: slog.debug( "(network.name: %s) setting lazy_blocked to false because instance on controller already exists" % network.name) lazy_blocked = False slog.debug("(network.name: %s) lazy_blocked: %s" % (network.name, lazy_blocked)) nd = ControllerNetwork(network=network, controller=expected_controller, lazy_blocked=lazy_blocked) slog.debug( "(network.name: %s) create ControllerNetwork [%s] with lazy_blocked [%s]" % (network.name, nd, lazy_blocked)) if network.subnet: # XXX: Possibly unpredictable behavior if there is # more than one ControllerNetwork and the subnet # is specified. slog.debug("(network.name: %s) network.subnet: %s" % (network.name, network.subnet)) nd.subnet = network.subnet nd.save()
def handle_create(self, network): slog.debug("network object: %s" % str(network)) return self.handle_update(network)
def update_instance(self, instance): slog.info( "[THREAD] instance_name: %s Thread: %s instance.updated: %s" % (instance.instance_name, threading.current_thread().name, instance.updated)) time.sleep(10) try: inst_id = instance.id slog.debug("[THREAD] inst_id : %s" % inst_id) old_update = 0 for idx in range(1, 100, 1): slog.debug("[THREAD] idx: %s" % idx) new_inst = Instance.objects.get(id=inst_id) if old_update == 0: old_update = new_inst.updated slog.debug("[THREAD] updated(old) : %s" % time.strftime( "%Y-%m-%d %H:%M:%S", time.gmtime(old_update))) if (new_inst.volumes is not None) and (new_inst.instance_name is not None): if (len(new_inst.volumes) > 2) and (len( new_inst.instance_name) > 2): slog.debug("[THREAD] instance_name : %s (%s)" % (new_inst.instance_name, len(new_inst.instance_name))) slog.debug("[THREAD] volumes : %s (%s)" % (new_inst.volumes, len(new_inst.volumes))) """ if idx == 1: self.update_instance_with_ssh(inst_id) """ new_inst.numberCores = idx new_inst.updated = time.time() slog.debug( "[THREAD] updated to renew : %s (%s)" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(new_inst.updated)), new_inst.updated)) new_inst.save(update_fields=['updated', 'numberCores']) time.sleep(1) clone_inst = Instance.objects.get(id=inst_id) clone_inst.save(update_fields=['numberCores']) slog.debug( "[THREAD] updated : %s (%s)" % (time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(clone_inst.updated)), clone_inst.updated)) if clone_inst.updated == old_update: slog.debug( "[THREAD] updated date was not changed. Nothing is executed. Waiting 5 seconds." ) time.sleep(5) else: slog.debug( "[THREAD] updated date was changed. Swarm synchronizer will run ansible" ) return except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc()))
def map_delete_inputs(self, instance): try: controller_register = json.loads( instance.node.site_deployment.controller.backend_register) slog.debug("controller_register: %s" % controller_register) if (controller_register.get('disabled', False)): slog.info('Controller %s is disabled' % instance.node.site_deployment.controller.name) raise InnocuousException( 'Controller %s is disabled' % instance.node.site_deployment.controller.name) instance_name = '%s-%s-%d' % (instance.slice.service.name, instance.slice.name, instance.id) slog.debug("instance_name: %s" % instance_name) controller = instance.node.site_deployment.controller slog.debug("controller: %s" % controller) swarm_manager_url = controller.auth_url slog.debug("swarm_manager_url: %s" % swarm_manager_url) (swarm_manager_address, docker_registry_port) = swarm_manager_url.split(':') slog.debug( "swarm_manager_address: %s docker_registry_port: %s" % (swarm_manager_address, docker_registry_port)) host_volume_path = "/opt/xos/instance_volume/%s" % instance.id slog.debug("host_volume_path: %s" % host_volume_path) input = { 'swarm_manager_address': swarm_manager_address, 'swarm_service_name': instance_name, 'host_volume_path': host_volume_path, 'ansible_tag': instance_name, 'delete': True } return input except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc()))
def map_sync_inputs(self, instance): try: slog.debug("instance: %s slice: %s" % (instance, instance.slice.name)) controller = instance.node.site_deployment.controller swarm_manager_url = controller.auth_url slog.info("swarm_manager_url: %s" % swarm_manager_url) (swarm_manager_address, docker_registry_port) = swarm_manager_url.split(':') slog.info("swarm_manager_address: %s docker_registry_port: %s" % (swarm_manager_address, docker_registry_port)) ## if instance.instance_uuid is not None, ## then This method will update the instance with new configuration. swarm_service_update_flag = False if instance.instance_uuid is not None: if len(instance.instance_uuid) > 2: swarm_service_update_flag = True # check if this service is created already on swarm cluster. duplicated_flag = self.chk_svc_exist(instance, swarm_manager_address) if duplicated_flag is True: swarm_service_update_flag = True ## To get volume information slog.debug("slice.mount_data_sets: %s" % instance.slice.mount_data_sets) mount_dest_path = "/usr/local/etc" if len(instance.slice.mount_data_sets) < 2: slog.debug("instance.slice.mount_data_sets(%s) is too short" % instance.slice.mount_data_sets) else: mount_dest_path = instance.slice.mount_data_sets slog.debug("volume mount destination path: %s" % mount_dest_path) ## set options for volume mounting ## (example) --mount type=bind,src=/opt/xos/instance_volume/1,dst=/usr/local/etc/haproxy volume_mount_opt = " " if swarm_service_update_flag is True: volume_mount_opt = "--mount-add type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % ( instance.id, mount_dest_path) else: volume_mount_opt = "--mount type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % ( instance.id, mount_dest_path) slog.debug("volume_mount_opt: %s" % volume_mount_opt) host_volume_path = "/opt/xos/instance_volume/%s" % instance.id slog.debug("host_volume_path: %s" % host_volume_path) # sanity check - make sure model_policy for slice has run if ((not instance.slice.policed) or (instance.slice.policed < instance.slice.updated)): slog.info( "Instance %s waiting on Slice %s to execute model policies" % (instance, instance.slice.name)) raise DeferredException( "Instance %s waiting on Slice %s to execute model policies" % (instance, instance.slice.name)) # sanity check - make sure model_policy for all slice networks have run networks = instance.slice.ownedNetworks.all() slog.debug("network list for slice of this instance(%s): %s" % (instance.name, str(networks))) for network in instance.slice.ownedNetworks.all(): slog.info("instance: %s network of slice(%s): %s" % (instance.name, instance.slice.name, network.name)) if ((not network.policed) or (network.policed < network.updated)): slog.info( "Instance %s waiting on Network %s to execute model policies" % (instance, network.name)) raise DeferredException( "Instance %s waiting on Network %s to execute model policies" % (instance, network.name)) slog.debug("Model Policy checking is done successfully.") swarm_network = "" for network in networks: slog.debug("networkd.id: %s(%s, %s) controller.id: %s" % (network.id, network.name, network.subnet, instance.node.site_deployment.controller.id)) if not ControllerNetwork.objects.filter( network_id=network.id, controller_id=instance.node.site_deployment.controller. id).exists(): raise DeferredException( "Instance %s Private Network %s lacks ControllerNetwork object" % (instance, network.name)) swarm_network += " --network %s " % network.name slog.debug("swarm_network: %s" % swarm_network) image_name = None controller_images = instance.image.controllerimages.all() controller_images = [ x for x in controller_images if x.controller_id == instance.node.site_deployment.controller.id ] if controller_images: image_name = controller_images[0].image.name slog.info("using image from ControllerImage object: " + str(image_name)) host_filter = instance.node.name.strip() slog.info("instance.node.name: %s" % instance.node.name) instance_name = '%s-%s-%d' % (instance.slice.service.name, instance.slice.name, instance.id) slog.info( "service name: %s instance.slice.name: %s instance.id: %s instance_name: %s" % (instance.slice.service.name, instance.slice.name, instance.id, instance_name)) self.instance_name = instance_name input_fields = { 'swarm_manager_address': swarm_manager_address, 'swarm_service_name': instance_name, 'network_name': swarm_network, 'replicas': "--replicas 1", 'restart_condition': "--restart-condition any --restart-delay 9s ", 'volume': volume_mount_opt, 'host_volume_path': host_volume_path, 'docker_registry_port': docker_registry_port, 'image_name': instance.image.name, 'image_tag': instance.image.tag, 'ansible_tag': instance_name, 'delete': False, 'update': swarm_service_update_flag } slog.info("input_fields: %s" % input_fields) if swarm_service_update_flag is False: slog.info( "swarm_service_update_flag is %s, so I will update once more" % swarm_service_update_flag) try: my_thr = threading.Thread(target=self.update_instance, args=(instance, )) my_thr.start() except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc())) return input_fields except Exception as ex: slog.error("Exception: %s %s %s" % (type(ex), str(ex), ex.args)) slog.error("%s" % str(traceback.format_exc()))