def check_instance_status():
while True:
time.sleep(5)
instances = Instance.objects.all()
slog.info("instances.count = %s" % len(instances))
for ins in instances:
tag = ""
provisioning_status = ""
try:
tag = Tag.objects.get(object_id=ins.id, name="chk_container_status")
except Exception as err:
slog.error("Error: object_id(%s) does not exist in Tag table (%s)" % (ins.id, str(err)))
continue
if ins.backend_code == 0:
provisioning_status = "PENDING_UPDATE"
elif ins.backend_code == 1:
if tag.value == "":
provisioning_status = "PENDING_UPDATE"
else:
try:
userdata = json.loads(tag.value)
create_timestamp = time.mktime(datetime.datetime.strptime(userdata['create_date'], "%Y-%m-%d %H:%M:%S").timetuple())
update_timestamp = time.mktime(datetime.datetime.strptime(userdata['update_date'], "%Y-%m-%d %H:%M:%S").timetuple())
if userdata['result'] == "Initialized":
provisioning_status = "PENDING_UPDATE"
elif userdata['expected_result'] != userdata['result'] and (float(update_timestamp) - float(create_timestamp)) > 30:
provisioning_status = "ERROR"
else:
provisioning_status = "ACTIVE"
except Exception as err:
slog.error("Error: json.loads() failed (%s)" % str(err))
else:
try:
userdata = json.loads(tag.value)
create_timestamp = time.mktime(datetime.datetime.strptime(userdata['create_date'], "%Y-%m-%d %H:%M:%S").timetuple())
update_timestamp = time.mktime(datetime.datetime.strptime(userdata['update_date'], "%Y-%m-%d %H:%M:%S").timetuple())
if (float(update_timestamp) - float(create_timestamp)) < 30:
provisioning_status = "PENDING_UPDATE"
else:
provisioning_status = "ERROR"
except Exception as err:
slog.error("Error: json.loads() failed (%s)" % str(err))
try:
lb = Loadbalancer.objects.get(tenantwithcontainer_ptr_id=ins.id)
lb.provisioning_status = provisioning_status
lb.save()
slog.info("id=%s, instance_name=%s, lb.provisioning_status=%s"
% (ins.id, ins.instance_name, lb.provisioning_status))
except Exception as err:
slog.error("Error: id(%s) does not exist in Loadbalancer table (%s)" % (ins.id, str(err)))
def update_lb_vip_addr(instance_id, vip_address):
try:
lb = Loadbalancer.objects.get(instance_id=instance_id)
lb.vip_address = vip_address
lb.save(update_fields=['vip_address', 'updated'], always_update_timestamp=True)
except Exception as err:
slog.error("%s" % str(err))
slog.info("lb.vip_address = %s" % lb.vip_address)
def update_loadbalancer_status(self, lb_id):
lb = Loadbalancer.objects.get(loadbalancer_id=lb_id)
if lb:
listeners = Listener.objects.filter(id=lb.listener_id)
for listener in listeners:
pools = Pool.objects.filter(id=lb.pool_id)
for pool in pools:
members = Member.objects.filter(memberpool_id=pool.id)
if len(members) > 0:
healths = Healthmonitor.objects.filter(
id=pool.health_monitor_id)
if len(healths) > 0:
lb.provisioning_status = "ACTIVE"
else:
slog.error(
"Healthmonitor information does not exist (id=%s)"
% pool.health_monitor_id)
lb.provisioning_status = "ERROR"
else:
slog.error(
"Member information does not exist (memberpool_id=%s)"
% pool.id)
lb.provisioning_status = "ERROR"
if len(pools) == 0:
slog.error(
"Pool information does not exist (loadbalancer_id=%s, id=%s)"
% (lb_id, lb.pool_id))
lb.provisioning_status = "ERROR"
if len(listeners) == 0:
slog.error("Listener information does not exist (id=%s)" %
lb.listener_id)
lb.provisioning_status = "ERROR"
else:
slog.error(
"Loadbalancer information does not exist (loadbalancer_id=%s)"
% lb_id)
lb.provisioning_status = "ERROR"
slog.info("lb.provisioning_status=%s" % lb.provisioning_status)
lb.save()
return lb.provisioning_status
def check_lb_vip_address():
while True:
time.sleep(5)
lbs_list = []
ports_list = []
lbs = Loadbalancer.objects.all()
slog.info("lbs.count = %s" % len(lbs))
for lb in lbs:
lb_info = {}
lb_info['id'] = lb.id
lb_info['instance_id'] = lb.id
lb_info['vip_address'] = lb.vip_address
slog.info("[Loadbalancer] lb.id=%s, lb.instance_id=%s, lb.vip=%s"
% (lb.id, lb.instance_id, lb.vip_address))
lbs_list.append(lb_info)
if len(lbs) == 0:
continue
ports = Port.objects.all()
slog.info("ports.count = %s" % len(ports))
for port in ports:
port_info = {}
port_info['instance_id'] = port.instance_id
port_info['ip'] = port.ip
slog.info("[Port] port.instance_id=%s, port.ip=%s" % (port.instance_id, port.ip))
ports_list.append(port_info)
for lb in lbs_list:
for port in ports_list:
if lb['instance_id'] == port['instance_id'] and lb['vip_address'] != port['ip']:
slog.info("instance_id=%s, lb.vip_address=%s, port.ip=%s"
% (lb['instance_id'], lb['vip_address'], port['ip']))
update_lb_vip_addr(lb['instance_id'], port['ip'])
def get_extra_attributes(self, o):
slog.info(
"===============================================================")
slog.info("instance_name=%s, instance_id=%d, instance_uuid=%s" %
(o.instance.instance_name, o.instance_id,
o.instance.instance_uuid))
try:
tags = Tag.objects.filter(object_id=o.instance.id)
if not len(tags):
userdata = {}
userdata['create_date'] = time.strftime(
"%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
userdata['update_date'] = time.strftime(
"%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
userdata['command'] = "service haproxy status"
userdata['expected_result'] = "haproxy is running."
userdata['result'] = "Initialized"
tag = Tag(service=o.instance.slice.service,
content_type=o.instance.self_content_type_id,
object_id=o.instance.id,
name="chk_container_status",
value=json.dumps(userdata))
tag.save()
except Exception as e:
slog.error("Instance.objects.get() failed - %s" % str(e))
lb_status = True
if self.update_pool_status(o.pool_id) != "ACTIVE":
slog.error("Pool status is not ACTIVE (pool_id=%s)" % o.pool_id)
lb_status = False
if self.update_loadbalancer_status(o.loadbalancer_id) != "ACTIVE":
slog.error(
"Loadbalancer status is not ACTIVE (loadbalancer_id=%s)" %
o.loadbalancer_id)
lb_status = False
if not lb_status:
return None
fields = {}
fields['instance_id'] = o.instance.id
fields['update_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
fields["baremetal_ssh"] = True
loadbalancer = {}
loadbalancer['loadbalancer_id'] = o.loadbalancer_id
loadbalancer['lb_name'] = o.name
loadbalancer['vip_address'] = o.vip_address
fields['loadbalancer'] = json.dumps(loadbalancer, indent=4)
slog.info(">>>>> Loadbalancer")
slog.info("%s" % json.dumps(loadbalancer, indent=4))
try:
listener = {}
obj = Listener.objects.get(id=o.listener_id)
listener['listener_name'] = obj.name
listener['listener_id'] = obj.listener_id
listener['protocol'] = obj.protocol
listener['protocol_port'] = obj.protocol_port
listener['stat_port'] = obj.stat_port
listener['connection_limit'] = obj.connection_limit
fields['listener'] = json.dumps(listener, indent=4)
slog.info(">>>>> Listener")
slog.info("%s" % json.dumps(listener, indent=4))
except Exception as e:
slog.error("Listener.objects.get() failed - %s" % str(e))
return None
try:
pool = {}
obj = Pool.objects.get(id=o.pool_id)
pool['pool_name'] = obj.name
pool['pool_id'] = obj.pool_id
pool['health_monitor_id'] = obj.health_monitor_id
pool['lb_algorithm'] = obj.lb_algorithm
pool['protocol'] = obj.protocol
fields['pool'] = json.dumps(pool, indent=4)
slog.info(">>>>> Pool")
slog.info("%s" % json.dumps(pool, indent=4))
except Exception as e:
slog.error("Pool.objects.get() failed - %s" % str(e))
return None
try:
root_obj = {}
member_list = []
root_obj['members'] = member_list
objs = Member.objects.filter(memberpool_id=o.pool_id)
for obj in objs:
member_obj = {}
member_obj['member_id'] = obj.member_id
member_obj['address'] = obj.address
member_obj['protocol_port'] = obj.protocol_port
member_obj['weight'] = obj.weight
member_list.append(member_obj)
fields['members'] = json.dumps(root_obj, indent=4)
slog.info(">>>>> Members")
slog.info("%s" % json.dumps(root_obj, indent=4))
except Exception as e:
slog.error("Member.objects.get() failed - %s" % str(e))
try:
health_monitor = {}
obj = Healthmonitor.objects.get(id=pool['health_monitor_id'])
health_monitor['health_monitor_id'] = obj.health_monitor_id
health_monitor['type'] = obj.type
health_monitor['delay'] = obj.delay
health_monitor['max_retries'] = obj.max_retries
health_monitor['timeout'] = obj.timeout
health_monitor['http_method'] = obj.http_method
health_monitor['url_path'] = obj.url_path
health_monitor['expected_codes'] = obj.expected_codes
fields['health_monitor'] = json.dumps(health_monitor, indent=4)
slog.info(">>>>> Healthmonitor")
slog.info("%s" % json.dumps(health_monitor, indent=4))
except Exception as e:
slog.error("Healthmonitor.objects.get() failed - %s" % str(e))
slog.info(
"===============================================================")
slog.info(">>> curl command for haproxy test")
slog.info("curl %s:%s" %
(loadbalancer['vip_address'], listener['protocol_port']))
fields = self.convert_unicode_to_str(fields)
return fields
slog.error("Error: id(%s) does not exist in Loadbalancer table (%s)" % (ins.id, str(err)))
if __name__ == "__main__":
models_active = False
wait = False
while not models_active:
try:
first_controller = Controller.objects.first()
slog.debug("one of controller set: %s" % first_controller.name)
first_image = Image.objects.first()
slog.debug("one of image set : %s" % first_image.name)
models_active = True
except Exception as e:
slog.info(str(e))
slog.info('Waiting for data model to come up before starting...')
time.sleep(3)
wait = True
slog.debug("Data Model is active (first_controller: %s)" % first_controller)
if wait:
time.sleep(5) # Safety factor, seeing that we stumbled waiting for the data model to come up.
lb_thr = threading.Thread(target=check_lb_vip_address)
lb_thr.start()
ins_thr = threading.Thread(target=check_instance_status)
ins_thr.start()