class Microscaler:
containers = [
{
"id": "/microscaling/analyzer",
"name": "analyzer-microscaling.marathon.slave.mesos"
},
{
"id": "/microscaling/batch",
"name": "batch-microscaling.marathon.slave.mesos"
}
]
cool_off_period = 10 # time to wait between scale requests
last_scale_up_time = time.time()
last_scale_down_time = time.time()
scale_factor = 5
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
def scaleUp(self, container):
"""Scale health indicates that we need to scale up, calculate the new
number of instances and issue the scale request."""
self.last_scale_up_time = time.time()
# Get current instance count
url = "http://leader.mesos:8080/v2/apps" + container["id"]
resp = requests.get(url)
app_data = json.loads(resp.text)
instances = app_data["app"]["instances"]
# Increment count
if container["id"] == "/microscaling/analyzer":
length = self.msgQueue.getLength()
new_instances = instances + 1 + int(length / 10)
else:
new_instances = instances + 1
self.scale(container["id"], new_instances)
def scaleDown(self, container):
"""Scale health indicates that we need to scale down so reduce the
number of instances by 1. """
self.last_scale_down_time = time.time()
# Get current instance count
url = "http://leader.mesos:8080/v2/apps" + container["id"]
resp = requests.get(url)
app_data = json.loads(resp.text)
instances = app_data["app"]["instances"]
# Decrement count
instances = instances - 1
if (instances <= 0):
return
self.scale(container["id"], instances, True)
def scale(self, container_id, instances, force = False ):
""" Scale a container to a given number of instances"""
url = "http://leader.mesos:8080/v2/apps" + container_id
if force:
url = url + "?force=true"
data_packet = { "instances": instances }
resp = requests.put(url, data = json.dumps(data_packet))
if resp.status_code == 200:
self.log.debug("Scaling to " + str(instances))
else:
self.log.debug("Problem scaling the container. Status code = " + str(resp.status_code))
def autoscale(self):
while True:
for container in self.containers:
try:
resp = requests.get("http://" + container["name"] + ":5000")
scale_health = json.loads(resp.text)
status = scale_health["status"]
now = time.time()
if status <= -100:
self.scaleDown(container)
elif status >= 100:
self.scaleUp(container)
else:
if self.last_scale_down_time < self.last_scale_up_time:
time_since_last_scale = now - self.last_scale_down_time
else:
time_since_last_scale = now - self.last_scale_up_time
if time_since_last_scale > self.cool_off_period:
if status >= 100:
self.scaleUp(container)
elif status < 0:
self.scaleDown(container)
except:
self.log.debug("Error checking scale health of " + container["id"])
time.sleep(5)
class Analyzer:
time_since_last_event = None
last_event_time = time.time()
current_length = 0
last_length = 0
max_length = 10
def __init__(self):
self.log = Log()
self.log.debug("Storage account for analyzer: {0}".format(
config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.current_length = self.msgQueue.getLength()
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME,
config.AZURE_STORAGE_ACCOUNT_KEY,
config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.debug("Sleep time between analyses: {0}".format(
self.sleep_time))
def incrementCount(self, event_type):
count = self.summary.getCount(event_type)
count = count + 1
self.summary.updateCount(event_type, count)
self.log.info(event_type + " count is now " + str(count))
def processEvent(self, message):
# Sleep to simulated a longer running process
time.sleep(self.sleep_time)
data = json.loads(message.message_text)
event_type = data["type"]
now = time.time() * 1000.0
duration = now - data["time"]
print("Duration of last event processing: " + str(duration))
self.incrementCount(event_type)
self.summary.updateLastProcessingTime(duration)
def fullAnalysis(self):
hostname = socket.gethostname()
msg = hostname + ': Analyzing log event queue'
notify.info(msg)
while True:
self.last_length = self.current_length
self.current_length = self.msgQueue.getLength()
if self.current_length > 0:
events = self.msgQueue.dequeue()
if len(events) > 0:
now = time.time()
for event in events:
self.time_since_last_event = now - self.last_event_time
self.last_event_time = now
self.log.info("Dequeued: " + event.message_text)
try:
self.processEvent(event)
self.msgQueue.delete(event)
self.current_length = self.current_length - 1
self.log.info("Counted and deleted: " +
event.message_text)
except:
e = sys.exc_info()
self.log.error("Could not process: " +
event.message_text +
" because %s" % e[0])
self.log.error(traceback.format_tb(e[2]))
time.sleep(self.sleep_time)