def initiate():
# Boot first machines if not active:
stack = openstack()
backends = stack.backends()
# Gathering first data
data = {}
data['acu'] = hastats.get_backend_cum_requests()['stot']
data['diff'] = 0
data['diffpt'] = 0
data['date'] = datetime.datetime.now()
data['active'] = len(
stack.active_backends()) #len(hastats.get_backends_up())
data['haactive'] = len(hastats.get_backends_up())
data['needed'] = None
data['epoch'] = (datetime.datetime.now() - epoch_start).seconds
metrics.append(data)
print metrics
time.sleep(sleeptime)
last = data
data = {}
data['acu'] = hastats.get_backend_cum_requests()['stot']
data['diff'] = int(
(float(data['acu']) - float(last['acu'])) / float(sleeptime))
data['diffpt'] = data['diff'] * sleeptime
data['date'] = datetime.datetime.now()
data['needed'] = needed_servers(acu=data['acu'], diff=data['diff'])
data['active'] = len(
stack.active_backends()) #len(hastats.get_backends_up())
data['haactive'] = len(hastats.get_backends_up())
data['epoch'] = (datetime.datetime.now() - epoch_start).seconds
metrics.append(data)
time.sleep(sleeptime)
def scale_down(Needed=1):
ha = haproxy.HAproxy()
stack = openstack()
active = stack.active_backends()
passive = stack.passive_backends()
instance = None
print "active servs: %s, min bakends: %s, needed: %s" % (str(len(active)), str(min_backends), str(Needed))
toremove = len(active) - Needed
removed = 0
threads = []
if len(passive) > 1:
# Delete the stopped nodes, and leave one
for node in passive[1:][::-1]:
handle_scaledown(node, delete=True)
if toremove > 0:
print "Want to remove %s nodes" % str(toremove)
if len(active) <= min_backends:
return False
elif (len(active) - toremove) > min_backends:
for i in range(1, toremove + 1):
if 'ACTIVE' in active[-i].status:
handle_scaledown(active[-i], stop=True)
else:
recalculate = len(active) - (Needed + min_backends)
for i in range(1, recalculate+1):
if 'ACTIVE' in active[-i].status:
handle_scaledown(active[-i], stop=True)
else:
print "No nodes to stop/delete"
return False
return True
def initiate():
# Boot first machines if not active:
stack = openstack()
backends = stack.backends()
# Gathering first data
data = {}
data['acu'] = hastats.get_backend_cum_requests()['stot']
data['diff'] = 0
data['diffpt'] = 0
data['date'] = datetime.datetime.now()
data['active'] = len(stack.active_backends())#len(hastats.get_backends_up())
data['haactive'] = len(hastats.get_backends_up())
data['needed'] = None
data['epoch'] = (datetime.datetime.now()-epoch_start).seconds
metrics.append(data)
print metrics
time.sleep(sleeptime)
last = data
data = {}
data['acu'] = hastats.get_backend_cum_requests()['stot']
data['diff'] = int((float(data['acu']) - float(last['acu'])) / float(sleeptime))
data['diffpt'] = data['diff'] * sleeptime
data['date'] = datetime.datetime.now()
data['needed'] = needed_servers(acu=data['acu'], diff=data['diff'])
data['active'] = len(stack.active_backends())#len(hastats.get_backends_up())
data['haactive'] = len(hastats.get_backends_up())
data['epoch'] = (datetime.datetime.now()-epoch_start).seconds
metrics.append(data)
time.sleep(sleeptime)
def update_conf():
""" Do we need to update the configuration? """
global ha_reloaded
stats = hastats.get_stat_backends()
stack = openstack()
backends = stack.backends()
if not len(backends) == len(stats):
ha = haproxy.HAproxy()
ha.compile(backends)
global ha_last_reload
ha_last_reload = datetime.datetime.now()
if ha.restart():
ha_reloaded = True
return True
return False
def update_conf():
""" Do we need to update the configuration? """
global ha_reloaded
stats = hastats.get_stat_backends()
stack = openstack()
backends = stack.backends()
if not len(backends) == len(stats):
ha = haproxy.HAproxy()
ha.compile(backends)
global ha_last_reload
ha_last_reload = datetime.datetime.now()
if ha.restart():
ha_reloaded = True
return True
return False
def main():
# Starting the first time
# getting current cum connections
try:
if not metrics:
print("Gathering initial data...")
# Gathering first data
initiate()
while True:
current = new_metrics(hastats.get_backend_cum_requests()['stot'])
print metrics[-1]
print "Needed servers: %s" % str(
needed_servers(diff=current['diff']))
# What to do? Scale up/down or are we happy?
stack = openstack()
active_backends = stack.active_backends()
up_backends = hastats.get_backends_up()
needed = needed_servers(diff=current['diff'])
if needed > len(active_backends):
print "Scaling up"
scale_up(needed - len(active_backends))
elif needed < len(active_backends):
print "Scaling down"
if not scale_down(Needed=needed):
print "Lowest number"
else:
# Sleeping
print "Sleeping one more round"
if update_conf():
print "HAproxy config reloaded"
print ha_last_reload
for line in hastats.get_stat_backends():
print line['svname'] + ', ' + line['status']
time.sleep(sleeptime)
except KeyboardInterrupt:
write_data()
def handle_scaledown(instance, delete=False, stop=False):
print "Starting to handle scaledown of %s" % instance.name
ha = haproxy.HAproxy()
stack = openstack()
# Set the instance in draining mode.
# No new conns. Active finishes
ha.drain(instance)
# Operating with short draintime (only static page)
time.sleep(1)
try:
if stop:
print "Stopping node %s" % instance.name
instance.stop()
elif delete:
print "Deleting node %s" % instance.name
instance.delete()
except:
print "Cant stop/delete instnace %s" % instance.name
traceback.print_exc(file=sys.stdout)
def handle_scaledown(instance, delete=False, stop=False):
print "Starting to handle scaledown of %s" % instance.name
ha = haproxy.HAproxy()
stack = openstack()
# Set the instance in draining mode.
# No new conns. Active finishes
ha.drain(instance)
# Operating with short draintime (only static page)
time.sleep(1)
try:
if stop:
print "Stopping node %s" % instance.name
instance.stop()
elif delete:
print "Deleting node %s" % instance.name
instance.delete()
except:
print "Cant stop/delete instnace %s" % instance.name
traceback.print_exc(file=sys.stdout)
def main():
# Starting the first time
# getting current cum connections
try:
if not metrics:
print("Gathering initial data...")
# Gathering first data
initiate()
while True:
current = new_metrics(hastats.get_backend_cum_requests()['stot'])
print metrics[-1]
print "Needed servers: %s" % str(needed_servers(diff=current['diff']))
# What to do? Scale up/down or are we happy?
stack = openstack()
active_backends = stack.active_backends()
up_backends = hastats.get_backends_up()
needed = needed_servers(diff=current['diff'])
if needed > len(active_backends):
print "Scaling up"
scale_up(needed-len(active_backends))
elif needed < len(active_backends):
print "Scaling down"
if not scale_down(Needed=needed):
print "Lowest number"
else:
# Sleeping
print "Sleeping one more round"
if update_conf():
print "HAproxy config reloaded"
print ha_last_reload
for line in hastats.get_stat_backends():
print line['svname'] + ', ' + line['status']
time.sleep(sleeptime)
except KeyboardInterrupt:
write_data()
def new_metrics(current_cumulated, hareset=False):
global ha_reloaded
global ha_last_reload
current = {}
current['acu'] = current_cumulated
current['date'] = datetime.datetime.now()
if ha_reloaded:
last_cumulated = 0
difference = int(ceil((float(current_cumulated) - float(last_cumulated)) \
/ float((current['date'] - ha_last_reload).seconds)))
diffpt = int(difference) * (current['date'] - ha_last_reload).seconds
try:
print "Current new cumulated connections: %s" % str(current_cumulated)
print "Calculation: float(%s) - float(%s) / float(%s-%s.seconds (%s))" % \
(str(current_cumulated), metrics[-1]['acu'], str(current['date']), str(metrics[-1]['date']),\
str((current['date'] - metrics[-1]['date']).seconds))
if ha_reloaded:
current['diff'] = difference
current['diffpt'] = diffpt
ha_reloaded = False
else:
current['diff'] = int(ceil((float(current_cumulated) - float(metrics[-1]['acu'])) \
/ float((current['date']-metrics[-1]['date']).seconds)))
current['diffpt'] = current['diff'] * (current['date'] -
metrics[-1]['date']).seconds
except ZeroDivisionError:
current['diff'] = 0
stack = openstack()
current['needed'] = needed_servers(acu=current['acu'],
diff=current['diff'])
current['active'] = len(
stack.active_backends()) #len(hastats.get_backends_up())
current['haactive'] = len(hastats.get_backends_up())
current['epoch'] = (datetime.datetime.now() - epoch_start).seconds
metrics.append(current)
return current
def new_metrics(current_cumulated, hareset=False):
global ha_reloaded
global ha_last_reload
current = {}
current['acu'] = current_cumulated
current['date'] = datetime.datetime.now()
if ha_reloaded:
last_cumulated = 0
difference = int(ceil((float(current_cumulated) - float(last_cumulated)) \
/ float((current['date'] - ha_last_reload).seconds)))
diffpt = int(difference) * (current['date'] - ha_last_reload).seconds
try:
print "Current new cumulated connections: %s" % str(current_cumulated)
print "Calculation: float(%s) - float(%s) / float(%s-%s.seconds (%s))" % \
(str(current_cumulated), metrics[-1]['acu'], str(current['date']), str(metrics[-1]['date']),\
str((current['date'] - metrics[-1]['date']).seconds))
if ha_reloaded:
current['diff'] = difference
current['diffpt'] = diffpt
ha_reloaded = False
else:
current['diff'] = int(ceil((float(current_cumulated) - float(metrics[-1]['acu'])) \
/ float((current['date']-metrics[-1]['date']).seconds)))
current['diffpt'] = current['diff'] * (current['date']-metrics[-1]['date']).seconds
except ZeroDivisionError:
current['diff'] = 0
stack = openstack()
current['needed'] = needed_servers(acu=current['acu'], diff=current['diff'])
current['active'] = len(stack.active_backends())#len(hastats.get_backends_up())
current['haactive'] = len(hastats.get_backends_up())
current['epoch'] = (datetime.datetime.now()-epoch_start).seconds
metrics.append(current)
return current
def scale_down(Needed=1):
ha = haproxy.HAproxy()
stack = openstack()
active = stack.active_backends()
passive = stack.passive_backends()
instance = None
print "active servs: %s, min bakends: %s, needed: %s" % (str(
len(active)), str(min_backends), str(Needed))
toremove = len(active) - Needed
removed = 0
threads = []
if len(passive) > 1:
# Delete the stopped nodes, and leave one
for node in passive[1:][::-1]:
handle_scaledown(node, delete=True)
if toremove > 0:
print "Want to remove %s nodes" % str(toremove)
if len(active) <= min_backends:
return False
elif (len(active) - toremove) > min_backends:
for i in range(1, toremove + 1):
if 'ACTIVE' in active[-i].status:
handle_scaledown(active[-i], stop=True)
else:
recalculate = len(active) - (Needed + min_backends)
for i in range(1, recalculate + 1):
if 'ACTIVE' in active[-i].status:
handle_scaledown(active[-i], stop=True)
else:
print "No nodes to stop/delete"
return False
return True
def scale_up(Number=1):
stack = openstack()
sleeping = stack.sleeping_machine()
if len(stack.backends()) == (quota_limit):
return False
scaled = 0
if sleeping:
print sleeping
if len(sleeping) > 1:
for node in sleeping:
if scaled < Number: # and not 'powering-on' in node.state:
node.start()
scaled += 1
else:
if not 'ACTIVE' in sleeping[0].status:
sleeping[0].start()
scaled += 1
else:
if scaled < Number:
thread = Thread(target=stack.create_multiple(Number - scaled))
thread.start()
def scale_up(Number=1):
stack = openstack()
sleeping = stack.sleeping_machine()
if len(stack.backends()) == (quota_limit):
return False
scaled = 0
if sleeping:
print sleeping
if len(sleeping) > 1:
for node in sleeping:
if scaled < Number: # and not 'powering-on' in node.state:
node.start()
scaled += 1
else:
if not 'ACTIVE' in sleeping[0].status:
sleeping[0].start()
scaled += 1
else:
if scaled < Number:
thread = Thread(target=stack.create_multiple(Number-scaled))
thread.start()