def map_sync_inputs(self, controller_user):
slog.debug("controller user: %r" % controller_user)
if not controller_user.controller.admin_user:
slog.info("controller %r has no admin_user, skipping" %
controller_user.controller)
return
# All users will have at least the 'user' role at their home site/tenant.
# We must also check if the user should have the admin role
roles = ['user']
# setup user home site roles at controller
if not controller_user.user.site:
raise Exception('Siteless user %s' % controller_user.user.email)
else:
swarm_manager_url = controller_user.controller.auth_url
slog.info("swarm_manager_url: %s" % swarm_manager_url)
(swarm_manager_address,
docker_registry_port) = swarm_manager_url.split(':')
slog.info("swarm_manager_address: %s docker_registry_port: %s" %
(swarm_manager_address, docker_registry_port))
user_fields = {
'swarm_manager_address':
swarm_manager_address,
'admin_user':
controller_user.controller.admin_user,
'ansible_tag':
'%s@%s' %
(controller_user.user.email.replace('@', '-at-'),
controller_user.controller.name) # name of ansible playbook
}
slog.info("user_fields: %s" % user_fields)
return user_fields
def chk_svc_exist(self, instance, swarm_manager_address):
duplicated_flag = False
try:
instance_name = '%s-%s-%d' % (instance.slice.service.name,
instance.slice.name, instance.id)
slog.debug("Service name to chkeck: %s" % instance_name)
import docker
docker_api_base_url = "tcp://%s:4243" % swarm_manager_address
slog.debug("Docker API Base URL: %s" % docker_api_base_url)
my_client = docker.DockerClient(base_url=docker_api_base_url)
swarm_svc = my_client.services.get(instance_name)
slog.debug("swarm_svc.id : %s" % swarm_svc.id)
slog.debug("swarm_svc.name : %s" % swarm_svc.name)
slog.debug("swarm_svc.attrs : %s" % swarm_svc.attrs)
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))
slog.info(
"There is no duplicated service name, I would create new service (%s)"
% instance_name)
duplicated_flag = False
else:
slog.info("There is duplicated service name (%s)" % instance_name)
duplicated_flag = True
return duplicated_flag
def map_sync_outputs(self, controller_image, res):
slog.debug("Ansible playbook result: %s" % str(res))
slog.debug("Ansible playbook result[4]['image']['Id']: %s" %
str(res[4]['image']['Id']))
controller_image.glance_image_id = str(res[4]['image']['Id'])
if len(controller_image.glance_image_id) > 2:
controller_image.backend_status = '1 - OK'
else:
controller_image.backend_status = '2 - Ansible playbook failure'
controller_image.save()
def alloc_subnet(self, uuid):
# 16 bits only
slog.debug("uuid: %s" % str(uuid))
uuid_masked = uuid & 0xffff
a = 10
b = uuid_masked >> 8
c = uuid_masked & 0xff
d = 0
cidr = '%d.%d.%d.%d/24' % (a, b, c, d)
slog.debug("cidr: %s" % str(cidr))
return cidr
def map_delete_inputs(self, controller_image):
try:
controller_register = json.loads(
instance.node.site_deployment.controller.backend_register)
slog.debug("controller_register: %s" % controller_register)
if (controller_register.get('disabled', False)):
slog.info('Controller %s is disabled' %
instance.node.site_deployment.controller.name)
raise InnocuousException(
'Controller %s is disabled' %
instance.node.site_deployment.controller.name)
swarm_manager_url = controller_image.controller.auth_url
(swarm_manager_address,
docker_registry_port) = swarm_manager_url.split(':')
slog.info("swarm_manager_address: %s docker_registry_port: %s" %
(swarm_manager_address, docker_registry_port))
input_fields = {
'swarm_manager_address':
swarm_manager_address,
'docker_registry_port':
docker_registry_port,
'image_file_path':
controller_image.image.path,
'image_dir':
os.path.dirname(controller_image.image.path),
'image_name':
controller_image.image.name,
'image_tag':
controller_image.image.tag,
'delete':
True,
'ansible_tag':
'%s@%s' %
(controller_image.image.name, controller_image.controller.name
) # name of ansible playbook
}
slog.info("input_fields: %s" % str(input_fields))
return input_fields
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))
def map_sync_outputs(self, controller_network, res):
slog.debug("ansible playbook ressult: %s" % str(res))
slog.debug("ansible playbook ressult[1][stdout]: %s" %
str(res[1]['stdout']))
res_stdout = res[1]['stdout']
json_content = json.loads(res_stdout)
slog.debug("json_content: %s" % str(json_content))
network_id = json_content[0]['Id']
slog.debug("network_id: %s" % str(network_id))
subnet_id = "%s-subnet-%s" % (network_id, self.cidr)
slog.debug("subnet_id: %s" % str(subnet_id))
controller_network.net_id = network_id
controller_network.subnet = self.cidr
controller_network.subnet_id = subnet_id
controller_network.backend_status = '1 - OK'
controller_network.save()
def update_instance_with_ssh(self, inst_id):
try:
import subprocess
import shlex
new_inst = Instance.objects.get(id=inst_id)
controller = new_inst.node.site_deployment.controller
swarm_manager_url = controller.auth_url
(swarm_manager_address,
docker_registry_port) = swarm_manager_url.split(':')
slog.info("swarm_manager_address: %s docker_registry_port: %s" %
(swarm_manager_address, docker_registry_port))
volume_mount_opt = "--mount-add type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % (
new_inst.id, new_inst.volumes)
## DOCKER SERVICE UPDATE
ssh_cmd = "ssh root@%s docker service update --force %s %s" % (
swarm_manager_address, volume_mount_opt,
new_inst.instance_name)
slog.debug("SSH CMD: %s" % ssh_cmd)
popen = subprocess.Popen(ssh_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdoutdata, stderrdata) = popen.communicate()
slog.debug("popen result: %s etc data: %s" %
(stdoutdata, stderrdata))
## DOCKER SERVICE STATUS
ssh_cmd = "ssh root@%s docker service ps %s" % (
swarm_manager_address, new_inst.instance_name)
slog.debug("SSH CMD: %s" % ssh_cmd)
popen = subprocess.Popen(ssh_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
(stdoutdata, stderrdata) = popen.communicate()
slog.debug("popen result: %s etc data: %s" %
(stdoutdata, stderrdata))
except Exception as ex:
slog.info("Exception: %s %s %s" % (type(ex), str(ex), ex.args))
slog.info("%s" % str(traceback.format_exc()))
return 0
def chk_net_exist(self, controller_network, swarm_manager_address):
duplicated_flag = False
try:
slog.debug("network_name to check: %s" %
controller_network.network.name)
import docker
docker_api_base_url = "tcp://%s:4243" % swarm_manager_address
slog.info("Docker API Base URL: %s" % docker_api_base_url)
my_client = docker.DockerClient(base_url=docker_api_base_url)
net = my_client.networks.get(controller_network.network.name)
slog.debug("network.id : %s" % net.id)
slog.debug("network.name : %s" % net.name)
slog.debug("network.attrs[IPAM][Config] : %s" %
net.attrs['IPAM']['Config'])
slog.debug("network.attrs : %s" % net.attrs)
except Exception as ex:
slog.info("Exception: %s" % str(ex.args))
slog.info("Exception: %s" % str(ex))
duplicated_flag = False # There is no same network.
else:
duplicated_flag = True # There is same network.
slog.debug("network duplicated_flag: %s" % duplicated_flag)
return duplicated_flag
def map_sync_inputs(self, controller_network):
# make sure to not sync a shared network
slog.debug("controller_network: %s" % str(controller_network))
slog.debug("controller_network.subnet: %s" % controller_network.subnet)
slog.debug("controller_network.network_id: %s" %
controller_network.network_id)
slog.debug("controller_network.network.name: %s" %
controller_network.network.name)
slog.debug(
"controller_network.network.template.shared_network_name: %s" %
controller_network.network.template.shared_network_name)
slog.debug(
"controller_network.network.template.shared_network_id : %s" %
controller_network.network.template.shared_network_id)
# If shared_network_name is kuryr/libnetwork2:latest --> then, discard following if-statement.
"""
if (controller_network.network.template.shared_network_name or controller_network.network.template.shared_network_id):
if controller_network.network.template.shared_network_name.__contains__("kuryr"):
slog.debug("This is kuryr network")
else:
slog.debug("SyncStep.SYNC_WITHOUT_RUNNING: %s" % str(SyncStep.SYNC_WITHOUT_RUNNING))
return SyncStep.SYNC_WITHOUT_RUNNING
"""
slog.debug("network owner: %s creator: %s" %
(controller_network.network.owner,
controller_network.network.owner.creator))
if controller_network.network.owner and controller_network.network.owner.creator:
return self.save_controller_network(controller_network)
else:
slog.info("I could not create network controller information %s" %
controller_network)
raise Exception('I could not create network controller %s' %
controller_network)
def handle_update(self, network):
slog.debug("network.name: %s id: %s" % (network.name, network.id))
expected_controllers = Controller.objects.all()
slog.debug("(network.name: %s) Controllers List: %s" %
(network.name, str(expected_controllers)))
existing_controllers = []
for cn in ControllerNetwork.objects.all():
slog.debug(
"(network.name: %s) controllernetwork.id: %s name: %s" %
(network.name, cn.id, cn.subnet))
slog.debug("(network.name: %s) controllernetwork.network.id: %s" %
(network.name, cn.network.id))
if cn.network.id == network.id:
existing_controllers.append(cn.controller)
existing_controller_ids = [c.id for c in existing_controllers]
slog.debug("(network.name: %s) existing_controller_id list: %s" %
(network.name, str(existing_controller_ids)))
for expected_controller in expected_controllers:
slog.debug("(network.name: %s) expected_controller: %s" %
(network.name, str(expected_controller)))
if expected_controller.id not in existing_controller_ids:
slog.debug("(network.name: %s) expected_controller.id: %s" %
(network.name, expected_controller.id))
lazy_blocked = False
# check and see if some instance already exists
for networkslice in network.networkslices.all():
slog.debug("(network.name: %s) networkslice: %s" %
(network.name, str(networkslice)))
found = False
for instance in networkslice.slice.instances.all():
slog.debug(
"(network.name: %s) instance.instance_name: %s" %
(network.name, instance_name))
slog.debug(
"(network.name: %s) instance.controller.id: %s expected_controller.id: %s"
% (network.name,
instance.node.site_deployment.controller.id,
expected_controller.id))
if instance.node.site_deployment.controller.id == expected_controller.id:
found = True
slog.debug("(network.name: %s) found: %s" %
(network.name, found))
if found:
slog.debug(
"(network.name: %s) setting lazy_blocked to false because instance on controller already exists"
% network.name)
lazy_blocked = False
slog.debug("(network.name: %s) lazy_blocked: %s" %
(network.name, lazy_blocked))
nd = ControllerNetwork(network=network,
controller=expected_controller,
lazy_blocked=lazy_blocked)
slog.debug(
"(network.name: %s) create ControllerNetwork [%s] with lazy_blocked [%s]"
% (network.name, nd, lazy_blocked))
if network.subnet:
# XXX: Possibly unpredictable behavior if there is
# more than one ControllerNetwork and the subnet
# is specified.
slog.debug("(network.name: %s) network.subnet: %s" %
(network.name, network.subnet))
nd.subnet = network.subnet
nd.save()
def handle_create(self, network):
slog.debug("network object: %s" % str(network))
return self.handle_update(network)
def update_instance(self, instance):
slog.info(
"[THREAD] instance_name: %s Thread: %s instance.updated: %s" %
(instance.instance_name, threading.current_thread().name,
instance.updated))
time.sleep(10)
try:
inst_id = instance.id
slog.debug("[THREAD] inst_id : %s" % inst_id)
old_update = 0
for idx in range(1, 100, 1):
slog.debug("[THREAD] idx: %s" % idx)
new_inst = Instance.objects.get(id=inst_id)
if old_update == 0:
old_update = new_inst.updated
slog.debug("[THREAD] updated(old) : %s" % time.strftime(
"%Y-%m-%d %H:%M:%S", time.gmtime(old_update)))
if (new_inst.volumes is not None) and (new_inst.instance_name
is not None):
if (len(new_inst.volumes) > 2) and (len(
new_inst.instance_name) > 2):
slog.debug("[THREAD] instance_name : %s (%s)" %
(new_inst.instance_name,
len(new_inst.instance_name)))
slog.debug("[THREAD] volumes : %s (%s)" %
(new_inst.volumes, len(new_inst.volumes)))
"""
if idx == 1:
self.update_instance_with_ssh(inst_id)
"""
new_inst.numberCores = idx
new_inst.updated = time.time()
slog.debug(
"[THREAD] updated to renew : %s (%s)" %
(time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(new_inst.updated)),
new_inst.updated))
new_inst.save(update_fields=['updated', 'numberCores'])
time.sleep(1)
clone_inst = Instance.objects.get(id=inst_id)
clone_inst.save(update_fields=['numberCores'])
slog.debug(
"[THREAD] updated : %s (%s)" %
(time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(clone_inst.updated)),
clone_inst.updated))
if clone_inst.updated == old_update:
slog.debug(
"[THREAD] updated date was not changed. Nothing is executed. Waiting 5 seconds."
)
time.sleep(5)
else:
slog.debug(
"[THREAD] updated date was changed. Swarm synchronizer will run ansible"
)
return
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))
def map_delete_inputs(self, instance):
try:
controller_register = json.loads(
instance.node.site_deployment.controller.backend_register)
slog.debug("controller_register: %s" % controller_register)
if (controller_register.get('disabled', False)):
slog.info('Controller %s is disabled' %
instance.node.site_deployment.controller.name)
raise InnocuousException(
'Controller %s is disabled' %
instance.node.site_deployment.controller.name)
instance_name = '%s-%s-%d' % (instance.slice.service.name,
instance.slice.name, instance.id)
slog.debug("instance_name: %s" % instance_name)
controller = instance.node.site_deployment.controller
slog.debug("controller: %s" % controller)
swarm_manager_url = controller.auth_url
slog.debug("swarm_manager_url: %s" % swarm_manager_url)
(swarm_manager_address,
docker_registry_port) = swarm_manager_url.split(':')
slog.debug(
"swarm_manager_address: %s docker_registry_port: %s" %
(swarm_manager_address, docker_registry_port))
host_volume_path = "/opt/xos/instance_volume/%s" % instance.id
slog.debug("host_volume_path: %s" % host_volume_path)
input = {
'swarm_manager_address': swarm_manager_address,
'swarm_service_name': instance_name,
'host_volume_path': host_volume_path,
'ansible_tag': instance_name,
'delete': True
}
return input
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))
def map_sync_inputs(self, instance):
try:
slog.debug("instance: %s slice: %s" %
(instance, instance.slice.name))
controller = instance.node.site_deployment.controller
swarm_manager_url = controller.auth_url
slog.info("swarm_manager_url: %s" % swarm_manager_url)
(swarm_manager_address,
docker_registry_port) = swarm_manager_url.split(':')
slog.info("swarm_manager_address: %s docker_registry_port: %s" %
(swarm_manager_address, docker_registry_port))
## if instance.instance_uuid is not None,
## then This method will update the instance with new configuration.
swarm_service_update_flag = False
if instance.instance_uuid is not None:
if len(instance.instance_uuid) > 2:
swarm_service_update_flag = True
# check if this service is created already on swarm cluster.
duplicated_flag = self.chk_svc_exist(instance,
swarm_manager_address)
if duplicated_flag is True:
swarm_service_update_flag = True
## To get volume information
slog.debug("slice.mount_data_sets: %s" %
instance.slice.mount_data_sets)
mount_dest_path = "/usr/local/etc"
if len(instance.slice.mount_data_sets) < 2:
slog.debug("instance.slice.mount_data_sets(%s) is too short" %
instance.slice.mount_data_sets)
else:
mount_dest_path = instance.slice.mount_data_sets
slog.debug("volume mount destination path: %s" % mount_dest_path)
## set options for volume mounting
## (example) --mount type=bind,src=/opt/xos/instance_volume/1,dst=/usr/local/etc/haproxy
volume_mount_opt = " "
if swarm_service_update_flag is True:
volume_mount_opt = "--mount-add type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % (
instance.id, mount_dest_path)
else:
volume_mount_opt = "--mount type=bind,src=/opt/xos/instance_volume/%s,dst=%s" % (
instance.id, mount_dest_path)
slog.debug("volume_mount_opt: %s" % volume_mount_opt)
host_volume_path = "/opt/xos/instance_volume/%s" % instance.id
slog.debug("host_volume_path: %s" % host_volume_path)
# sanity check - make sure model_policy for slice has run
if ((not instance.slice.policed)
or (instance.slice.policed < instance.slice.updated)):
slog.info(
"Instance %s waiting on Slice %s to execute model policies"
% (instance, instance.slice.name))
raise DeferredException(
"Instance %s waiting on Slice %s to execute model policies"
% (instance, instance.slice.name))
# sanity check - make sure model_policy for all slice networks have run
networks = instance.slice.ownedNetworks.all()
slog.debug("network list for slice of this instance(%s): %s" %
(instance.name, str(networks)))
for network in instance.slice.ownedNetworks.all():
slog.info("instance: %s network of slice(%s): %s" %
(instance.name, instance.slice.name, network.name))
if ((not network.policed)
or (network.policed < network.updated)):
slog.info(
"Instance %s waiting on Network %s to execute model policies"
% (instance, network.name))
raise DeferredException(
"Instance %s waiting on Network %s to execute model policies"
% (instance, network.name))
slog.debug("Model Policy checking is done successfully.")
swarm_network = ""
for network in networks:
slog.debug("networkd.id: %s(%s, %s) controller.id: %s" %
(network.id, network.name, network.subnet,
instance.node.site_deployment.controller.id))
if not ControllerNetwork.objects.filter(
network_id=network.id,
controller_id=instance.node.site_deployment.controller.
id).exists():
raise DeferredException(
"Instance %s Private Network %s lacks ControllerNetwork object"
% (instance, network.name))
swarm_network += " --network %s " % network.name
slog.debug("swarm_network: %s" % swarm_network)
image_name = None
controller_images = instance.image.controllerimages.all()
controller_images = [
x for x in controller_images if x.controller_id ==
instance.node.site_deployment.controller.id
]
if controller_images:
image_name = controller_images[0].image.name
slog.info("using image from ControllerImage object: " +
str(image_name))
host_filter = instance.node.name.strip()
slog.info("instance.node.name: %s" % instance.node.name)
instance_name = '%s-%s-%d' % (instance.slice.service.name,
instance.slice.name, instance.id)
slog.info(
"service name: %s instance.slice.name: %s instance.id: %s instance_name: %s"
% (instance.slice.service.name, instance.slice.name,
instance.id, instance_name))
self.instance_name = instance_name
input_fields = {
'swarm_manager_address': swarm_manager_address,
'swarm_service_name': instance_name,
'network_name': swarm_network,
'replicas': "--replicas 1",
'restart_condition':
"--restart-condition any --restart-delay 9s ",
'volume': volume_mount_opt,
'host_volume_path': host_volume_path,
'docker_registry_port': docker_registry_port,
'image_name': instance.image.name,
'image_tag': instance.image.tag,
'ansible_tag': instance_name,
'delete': False,
'update': swarm_service_update_flag
}
slog.info("input_fields: %s" % input_fields)
if swarm_service_update_flag is False:
slog.info(
"swarm_service_update_flag is %s, so I will update once more"
% swarm_service_update_flag)
try:
my_thr = threading.Thread(target=self.update_instance,
args=(instance, ))
my_thr.start()
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))
return input_fields
except Exception as ex:
slog.error("Exception: %s %s %s" %
(type(ex), str(ex), ex.args))
slog.error("%s" % str(traceback.format_exc()))