def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.HALO_QUEUE_MATCH)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME,
config.AZURE_STORAGE_ACCOUNT_KEY,
config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
def __init__(self):
self.log = Log()
self.log.debug("Storage account for analyzer: {0}".format(config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.keep_running = bool(config.ANALYZER_KEEP_RUNNING)
self.log.debug("Analyzer keep running: {0}".format(self.keep_running))
if self.keep_running:
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.debug("Sleep time between analyses: {0}".format(self.sleep_time))
class MatchAnalyzer:
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.HALO_QUEUE_MATCH)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME,
config.AZURE_STORAGE_ACCOUNT_KEY,
config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
def processMatch(self, match):
gamertag = match["Players"][0]["Player"]["Gamertag"]
result_code = match["Players"][0]["Result"]
if result_code == 3:
result = 'won'
elif result_code == 2:
result = 'tied'
elif result_code == 1:
result = 'lost'
else:
result = 'DNF'
count_type = gamertag + 'match' + result
count = self.summary.getCount(count_type)
count = count + 1
self.summary.updateCount(count_type, count)
print(count_type + ' is now ' + str(count))
def fullAnalysis(self):
while True:
matches = self.msgQueue.dequeue()
if len(matches) > 0:
for match_event_raw in matches:
match_event = json.loads(match_event_raw.message_text)
match = json.loads(match_event["data"])
# print("Processing:\n" + json.dumps(match, sort_keys=True, indent=4))
try:
self.processMatch(match)
self.msgQueue.delete(match_event_raw)
except:
e = sys.exc_info()[0]
self.log.error("Could not process: " +
match_event_raw.message_text +
" because %s" % e)
traceback.print_exc(file=sys.stdout)
else:
break
def getPlayerStats(self, gamertag):
stats = {}
stats.player = {"gamertag": gamertag}
count_type = gamertag + "won"
count = self.summary, getCount(count_type)
print(str(count), wins)
class Analyzer:
def __init__(self):
self.log = Log()
self.log.debug("Storage account for analyzer: {0}".format(config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.keep_running = bool(config.ANALYZER_KEEP_RUNNING)
self.log.debug("Analyzer keep running: {0}".format(self.keep_running))
if self.keep_running:
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.debug("Sleep time between analyses: {0}".format(self.sleep_time))
def incrementCount(self, event_type):
count = self.summary.getCount(event_type)
count = count + 1
self.summary.updateCount(event_type, count)
self.log.info(event_type + " count is now " + str(count))
def processEvent(self, message):
msg = message.message_text
split = msg.find(" - ")
if (not split):
event_type = "OTHER"
else:
event_type = msg[:split]
# Sleep to simulated a longer running process
time.sleep(self.sleep_time)
self.incrementCount(event_type)
def fullAnalysis(self):
hostname = socket.gethostname()
msg = hostname + ': Analyzing log event queue'
notify.info(msg)
while True:
events = self.msgQueue.dequeue()
if len(events) > 0:
for event in events:
self.log.info("Dequeued: " + event.message_text)
try:
self.processEvent(event)
self.msgQueue.delete(event)
self.log.info("Counted and deleted: " + event.message_text)
except:
e = sys.exc_info()[0]
self.log.error("Could not process: " + event.message_text + " because %s" % e)
traceback.print_exc(file=sys.stdout)
else:
break
time.sleep(self.sleep_time)
def __init__(self):
self.log = Log()
self.log.debug("Storage account for analyzer: {0}".format(
config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.current_length = self.msgQueue.getLength()
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME,
config.AZURE_STORAGE_ACCOUNT_KEY,
config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.debug("Sleep time between analyses: {0}".format(
self.sleep_time))
def create_queue_entries_for_match(match):
"""
Create all required queue entries for the supplied match.
"""
# Increment
work_item = {
"created": time.time(),
"data": json.dumps(match)
}
queue_name = config.HALO_QUEUE_MATCH
work_queue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=queue_name)
work_queue.enqueue(json.dumps(work_item))
def enqueue():
queue = request.form['queue']
msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=queue)
msg = request.form['message']
msgQueue.enqueue(msg)
response = {
"result": "success",
"message": msg,
"queue": queue,
"storage_account": config.AZURE_STORAGE_ACCOUNT_NAME
}
return jsonify(response)
class MatchAnalyzer:
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.HALO_QUEUE_MATCH)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
def processMatch(self, match):
gamertag = match["Players"][0]["Player"]["Gamertag"]
result_code = match["Players"][0]["Result"]
if result_code == 3:
result = 'won'
elif result_code == 2:
result = 'tied'
elif result_code == 1:
result = 'lost'
else:
result = 'DNF'
count_type = gamertag + 'match' + result
count = self.summary.getCount(count_type)
count = count + 1
self.summary.updateCount(count_type, count)
print(count_type + ' is now ' + str(count))
def fullAnalysis(self):
while True:
matches = self.msgQueue.dequeue()
if len(matches) > 0:
for match_event_raw in matches:
match_event = json.loads(match_event_raw.message_text)
match = json.loads(match_event["data"])
# print("Processing:\n" + json.dumps(match, sort_keys=True, indent=4))
try:
self.processMatch(match)
self.msgQueue.delete(match_event_raw)
except:
e = sys.exc_info()[0]
self.log.error("Could not process: " + match_event_raw.message_text + " because %s" % e)
traceback.print_exc(file=sys.stdout)
else:
break
def getPlayerStats(self, gamertag):
stats = {}
stats.player = { "gamertag": gamertag }
count_type = gamertag + "won"
count = self.summary,getCount(count_type)
print(str(count), wins)
class Analyzer:
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
def incrementCount(self, event_type):
count = self.summary.getCount(event_type)
count = count + 1
self.summary.updateCount(event_type, count)
self.log.info(event_type + " count is now " + str(count))
def processEvent(self, message):
msg = message.message_text
if msg.startswith("ERROR"):
event_type = "ERROR"
elif msg.startswith("WARNING"):
event_type = "WARNING"
elif msg.startswith("INFO"):
event_type = "INFO"
elif msg.startswith("DEBUG"):
event_type = "DEBUG"
else:
event_type = "OTHER"
self.incrementCount(event_type)
def fullAnalysis(self):
hostname = socket.gethostname()
msg = hostname + ': Analyzing log event queue'
notify.info(msg)
while True:
events = self.msgQueue.dequeue()
if len(events) > 0:
for event in events:
self.log.info("Dequeued: " + event.message_text)
try:
self.processEvent(event)
self.msgQueue.delete(event)
self.log.info("Counted and deleted: " + event.message_text)
except:
e = sys.exc_info()[0]
self.log.error("Could not process: " + event.message_text + " because %s" % e)
traceback.print_exc(file=sys.stdout)
else:
break
def enqueue():
queue = request.form['queue']
msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=queue)
msg = request.form['message']
msgQueue.enqueue(msg)
response = {
"result": "success",
"message": msg,
"queue": queue,
"storage_account": config.AZURE_STORAGE_ACCOUNT_NAME
}
return jsonify(response)
def __init__(self):
self.log = Log()
self.log.info("Storage account: {0}".format(config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.keep_running = bool(config.ANALYZER_KEEP_RUNNING)
self.log.info("Keep running: {0}".format(self.keep_running))
if self.keep_running:
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.info("Sleep time between analyses: {0}".format(self.sleep_time))
def getQueueService():
queue_service = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.AZURE_STORAGE_QUEUE_NAME)
return queue_service
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
class Microscaler:
containers = [
{
"id": "/microscaling/analyzer",
"name": "analyzer-microscaling.marathon.slave.mesos"
},
{
"id": "/microscaling/batch",
"name": "batch-microscaling.marathon.slave.mesos"
}
]
cool_off_period = 10 # time to wait between scale requests
last_scale_up_time = time.time()
last_scale_down_time = time.time()
scale_factor = 5
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
def scaleUp(self, container):
"""Scale health indicates that we need to scale up, calculate the new
number of instances and issue the scale request."""
self.last_scale_up_time = time.time()
# Get current instance count
url = "http://leader.mesos:8080/v2/apps" + container["id"]
resp = requests.get(url)
app_data = json.loads(resp.text)
instances = app_data["app"]["instances"]
# Increment count
if container["id"] == "/microscaling/analyzer":
length = self.msgQueue.getLength()
new_instances = instances + 1 + int(length / 10)
else:
new_instances = instances + 1
self.scale(container["id"], new_instances)
def scaleDown(self, container):
"""Scale health indicates that we need to scale down so reduce the
number of instances by 1. """
self.last_scale_down_time = time.time()
# Get current instance count
url = "http://leader.mesos:8080/v2/apps" + container["id"]
resp = requests.get(url)
app_data = json.loads(resp.text)
instances = app_data["app"]["instances"]
# Decrement count
instances = instances - 1
if (instances <= 0):
return
self.scale(container["id"], instances, True)
def scale(self, container_id, instances, force = False ):
""" Scale a container to a given number of instances"""
url = "http://leader.mesos:8080/v2/apps" + container_id
if force:
url = url + "?force=true"
data_packet = { "instances": instances }
resp = requests.put(url, data = json.dumps(data_packet))
if resp.status_code == 200:
self.log.debug("Scaling to " + str(instances))
else:
self.log.debug("Problem scaling the container. Status code = " + str(resp.status_code))
def autoscale(self):
while True:
for container in self.containers:
try:
resp = requests.get("http://" + container["name"] + ":5000")
scale_health = json.loads(resp.text)
status = scale_health["status"]
now = time.time()
if status <= -100:
self.scaleDown(container)
elif status >= 100:
self.scaleUp(container)
else:
if self.last_scale_down_time < self.last_scale_up_time:
time_since_last_scale = now - self.last_scale_down_time
else:
time_since_last_scale = now - self.last_scale_up_time
if time_since_last_scale > self.cool_off_period:
if status >= 100:
self.scaleUp(container)
elif status < 0:
self.scaleDown(container)
except:
self.log.debug("Error checking scale health of " + container["id"])
time.sleep(5)
def getMessageQueue(self, queue_id):
self.queue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=queue_id)
def simulate():
log = Log()
hostname = socket.gethostname()
msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.AZURE_STORAGE_QUEUE_NAME)
if int(config.SIMULATION_ACTIONS) > 0:
msg = hostname + ': Attempting to simulate ' + str(
config.SIMULATION_ACTIONS) + ' actions'
log.debug(msg)
notify.info(msg)
else:
msg = hostname + ': Simulating until stopped'
log.debug(msg)
notify.info(msg)
temp = 70
_actions = 0
while int(config.SIMULATION_ACTIONS) == 0 or int(
config.SIMULATION_ACTIONS) - _actions > 0:
change = random.randint(-1, 1)
if temp <= _too_cold:
change = 1
elif temp >= _too_hot:
change = -1
msgQueue.enqueue("INFO - Change since last reading: " + str(change))
temp = temp + change
msgQueue.enqueue("INFO - Current temperature: " + str(temp))
if temp == _just_right:
msgQueue.enqueue("INFO - That's perfect")
elif temp < _just_right and temp > _too_cold:
msgQueue.enqueue('WARNING - Getting a little chilly')
elif temp > _just_right and temp < _too_hot:
msgQueue.enqueue('WARNING - Getting a touch warm')
elif temp <= _too_cold:
msgQueue.enqueue('ERROR - Too cold, how did this happen?')
elif temp >= _too_hot:
msgQueue.enqueue('ERROR - Too hot, how did this happen?')
else:
msgQueue.enqueue('ERROR - Can' 't tell if it' 's hot or cold')
msgQueue.close()
_actions = _actions + 1
time.sleep(int(config.SIMULATION_DELAY))
msg = hostname + ": Simulated " + str(
config.SIMULATION_ACTIONS) + " actions and added them to the queue"
log.debug(msg)
notify.info(msg)
def simulate():
log = Log()
hostname = socket.gethostname()
msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.AZURE_STORAGE_QUEUE_NAME)
if int(config.SIMULATION_ACTIONS) > 0:
msg = hostname + ': Attempting to simulate ' + str(config.SIMULATION_ACTIONS) + ' actions'
log.debug(msg)
notify.info(msg)
else:
msg = hostname + ': Simulating until stopped'
log.debug(msg)
notify.info(msg)
temp = 70;
_actions = 0
while int(config.SIMULATION_ACTIONS) == 0 or int(config.SIMULATION_ACTIONS) - _actions > 0:
change = random.randint(-1, 1)
if temp <= _too_cold:
change = 1
elif temp >= _too_hot:
change = -1
msgQueue.enqueue("Change since last reading: " + str(change), level="INFO")
temp = temp + change
msgQueue.enqueue("Current temperature: " + str(temp), level="INFO")
if temp == _just_right:
msgQueue.enqueue("That's perfect", level="INFO")
elif temp < _just_right and temp > _too_cold:
msgQueue.enqueue('Getting a little chilly', level="WARNING")
elif temp > _just_right and temp < _too_hot:
msgQueue.enqueue('Getting a touch warm', level="WARNING")
elif temp <= _too_cold:
msgQueue.enqueue('Too cold, how did this happen?', level="ERROR")
elif temp >= _too_hot:
msgQueue.enqueue('Too hot, how did this happen?', level="ERROR")
else:
msgQueue.enqueue('Can''t tell if it''s hot or cold', level="ERROR")
msgQueue.close()
_actions = _actions + 1
time.sleep(int(config.SIMULATION_DELAY))
msg = hostname + ": Simulated " + str(config.SIMULATION_ACTIONS) + " actions and added them to the queue"
log.debug(msg)
notify.info(msg)
def __init__(self):
self.log = Log()
self.msgQueue = Queue(account_name = config.AZURE_STORAGE_ACCOUNT_NAME, account_key=config.AZURE_STORAGE_ACCOUNT_KEY, queue_name=config.HALO_QUEUE_MATCH)
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME, config.AZURE_STORAGE_ACCOUNT_KEY, config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
class Analyzer:
time_since_last_event = None
last_event_time = time.time()
current_length = 0
last_length = 0
max_length = 10
def __init__(self):
self.log = Log()
self.log.debug("Storage account for analyzer: {0}".format(
config.AZURE_STORAGE_ACCOUNT_NAME))
self.msgQueue = Queue(account_name=config.AZURE_STORAGE_ACCOUNT_NAME,
account_key=config.AZURE_STORAGE_ACCOUNT_KEY,
queue_name=config.AZURE_STORAGE_QUEUE_NAME)
self.current_length = self.msgQueue.getLength()
self.summary = SummaryTable(config.AZURE_STORAGE_ACCOUNT_NAME,
config.AZURE_STORAGE_ACCOUNT_KEY,
config.AZURE_STORAGE_SUMMARY_TABLE_NAME)
self.sleep_time = float(config.ANALYZER_SLEEP_TIME)
self.log.debug("Sleep time between analyses: {0}".format(
self.sleep_time))
def incrementCount(self, event_type):
count = self.summary.getCount(event_type)
count = count + 1
self.summary.updateCount(event_type, count)
self.log.info(event_type + " count is now " + str(count))
def processEvent(self, message):
# Sleep to simulated a longer running process
time.sleep(self.sleep_time)
data = json.loads(message.message_text)
event_type = data["type"]
now = time.time() * 1000.0
duration = now - data["time"]
print("Duration of last event processing: " + str(duration))
self.incrementCount(event_type)
self.summary.updateLastProcessingTime(duration)
def fullAnalysis(self):
hostname = socket.gethostname()
msg = hostname + ': Analyzing log event queue'
notify.info(msg)
while True:
self.last_length = self.current_length
self.current_length = self.msgQueue.getLength()
if self.current_length > 0:
events = self.msgQueue.dequeue()
if len(events) > 0:
now = time.time()
for event in events:
self.time_since_last_event = now - self.last_event_time
self.last_event_time = now
self.log.info("Dequeued: " + event.message_text)
try:
self.processEvent(event)
self.msgQueue.delete(event)
self.current_length = self.current_length - 1
self.log.info("Counted and deleted: " +
event.message_text)
except:
e = sys.exc_info()
self.log.error("Could not process: " +
event.message_text +
" because %s" % e[0])
self.log.error(traceback.format_tb(e[2]))
time.sleep(self.sleep_time)
