class Microscaler: containers = [ { "id": "/microscaling/analyzer", "name": "analyzer-microscaling.marathon.slave.mesos" }, { "id": "/microscaling/batch", "name": "batch-microscaling.marathon.slave.mesos" } ] cool_off_period = 10 # time to wait between scale requests last_scale_up_time = time.time() last_scale_down_time = time.time() scale_factor = 5 def __init__(self): self.log = Log() self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME) def scaleUp(self, container): """Scale health indicates that we need to scale up, calculate the new number of instances and issue the scale request.""" self.last_scale_up_time = time.time() # Get current instance count url = "http://leader.mesos:8080/v2/apps" + container["id"] resp = requests.get(url) app_data = json.loads(resp.text) instances = app_data["app"]["instances"] # Increment count if container["id"] == "/microscaling/analyzer": length = self.msgQueue.getLength() new_instances = instances + 1 + int(length / 10) else: new_instances = instances + 1 self.scale(container["id"], new_instances) def scaleDown(self, container): """Scale health indicates that we need to scale down so reduce the number of instances by 1. """ self.last_scale_down_time = time.time() # Get current instance count url = "http://leader.mesos:8080/v2/apps" + container["id"] resp = requests.get(url) app_data = json.loads(resp.text) instances = app_data["app"]["instances"] # Decrement count instances = instances - 1 if (instances <= 0): return self.scale(container["id"], instances, True) def scale(self, container_id, instances, force = False ): """ Scale a container to a given number of instances""" url = "http://leader.mesos:8080/v2/apps" + container_id if force: url = url + "?force=true" data_packet = { "instances": instances } resp = requests.put(url, data = json.dumps(data_packet)) if resp.status_code == 200: self.log.debug("Scaling to " + str(instances)) else: self.log.debug("Problem scaling the container. Status code = " + str(resp.status_code)) def autoscale(self): while True: for container in self.containers: try: resp = requests.get("http://" + container["name"] + ":5000") scale_health = json.loads(resp.text) status = scale_health["status"] now = time.time() if status <= -100: self.scaleDown(container) elif status >= 100: self.scaleUp(container) else: if self.last_scale_down_time < self.last_scale_up_time: time_since_last_scale = now - self.last_scale_down_time else: time_since_last_scale = now - self.last_scale_up_time if time_since_last_scale > self.cool_off_period: if status >= 100: self.scaleUp(container) elif status < 0: self.scaleDown(container) except: self.log.debug("Error checking scale health of " + container["id"]) time.sleep(5)
class Analyzer: time_since_last_event = None last_event_time = time.time() current_length = 0 last_length = 0 max_length = 10 def __init__(self): self.log = Log() self.log.debug("Storage account for analyzer: {0}".format( config.AZURE_STORAGE_ACCOUNT_NAME)) self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME) self.current_length = self.msgQueue.getLength() self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME) self.sleep_time = float(config.ANALYZER_SLEEP_TIME) self.log.debug("Sleep time between analyses: {0}".format( self.sleep_time)) def incrementCount(self, event_type): count = self.summary.getCount(event_type) count = count + 1 self.summary.updateCount(event_type, count) self.log.info(event_type + " count is now " + str(count)) def processEvent(self, message): # Sleep to simulated a longer running process time.sleep(self.sleep_time) data = json.loads(message.message_text) event_type = data["type"] now = time.time() * 1000.0 duration = now - data["time"] print("Duration of last event processing: " + str(duration)) self.incrementCount(event_type) self.summary.updateLastProcessingTime(duration) def fullAnalysis(self): hostname = socket.gethostname() msg = hostname + ': Analyzing log event queue' notify.info(msg) while True: self.last_length = self.current_length self.current_length = self.msgQueue.getLength() if self.current_length > 0: events = self.msgQueue.dequeue() if len(events) > 0: now = time.time() for event in events: self.time_since_last_event = now - self.last_event_time self.last_event_time = now self.log.info("Dequeued: " + event.message_text) try: self.processEvent(event) self.msgQueue.delete(event) self.current_length = self.current_length - 1 self.log.info("Counted and deleted: " + event.message_text) except: e = sys.exc_info() self.log.error("Could not process: " + event.message_text + " because %s" % e[0]) self.log.error(traceback.format_tb(e[2])) time.sleep(self.sleep_time)