def update_params(self, d): super(StompWidget, self).update_params(d) d.topics = [] d.onmessageframe = defaultdict(str) # {topic: 'js callbacks'} for callback in self.callbacks: if len(moksha.livewidgets[callback]): cbs = '' if callback == 'onmessageframe': for topic in moksha.livewidgets[callback]: d.topics.append(topic) for cb in moksha.livewidgets[callback][topic]: d.onmessageframe[topic] += '%s;' % str(cb) else: for cb in moksha.livewidgets[callback]: if isinstance(cb, (js_callback, js_function)): cbs += '$(%s);' % str(cb) else: cbs += str(cb) if cbs: d[callback] = cbs if d.notify: moksha_notify.register_resources() d.onopen = js_callback( 'function() { $.jGrowl("Moksha live socket connected") }') d.onerror = js_callback( 'function(error) { $.jGrowl("Moksha Live Socket Error: " + error) }' ) d.onerrorframe = js_callback( 'function(f) { $.jGrowl("Error frame received from Moksha Socket: " + f) }' ) d.onclose = js_callback( 'function(c) { $.jGrowl("Moksha Socket Closed") }')
def update_params(self, d): super(StompWidget, self).update_params(d) d.topics = [] d.onmessageframe = defaultdict(str) # {topic: 'js callbacks'} for callback in self.callbacks: if len(moksha.livewidgets[callback]): cbs = '' if callback == 'onmessageframe': for topic in moksha.livewidgets[callback]: d.topics.append(topic) for cb in moksha.livewidgets[callback][topic]: d.onmessageframe[topic] += '%s;' % str(cb) else: for cb in moksha.livewidgets[callback]: if isinstance(cb, (js_callback, js_function)): cbs += '$(%s);' % str(cb) else: cbs += str(cb) if cbs: d[callback] = cbs if d.notify: moksha_notify.register_resources() d.onopen = js_callback('function() { $.jGrowl("Moksha live socket connected") }') d.onerror = js_callback('function(error) { $.jGrowl("Moksha Live Socket Error: " + error) }') d.onerrorframe = js_callback('function(f) { $.jGrowl("Error frame received from Moksha Socket: " + f) }') d.onclose = js_callback('function(c) { $.jGrowl("Moksha Socket Closed") }')
def crawl_moksha_topics(self, entry_point='moksha.widget'): """ Iterate over all moksha widgets looking for topics """ topics = defaultdict(list) for entry in pkg_resources.iter_entry_points(entry_point): entry_class = entry.load() for topic in listify(getattr(entry_class, 'topic', [])): topics[to_unicode(topic)].append(entry_class) for topic in listify(getattr(entry_class, 'topics', [])): topics[to_unicode(topic)].append(entry_class) return topics
def index(self): #snake = Entity(u'snake') #snake[u'family'] = u'Python' #DBSession.add(snake) #DBSession.flush() num = DBSession.query(Entity).count() topics = defaultdict(list) for entity in DBSession.query(Entity).all(): topics[entity.name] = None # topics[entity.name] = entity.facts return dict(num=num, topics=topics)
def __init__(self): log.info('Loading the Moksha Hub') self.topics = defaultdict(list) self.__init_consumers() MokshaHub.__init__(self) if self.amqp_broker: self.__init_amqp() self.__run_consumers() self.__init_producers()
def register_livewidgets(self, environ): """ Register the `moksha.livewidgets` dictionary. This is a per-request StackedObjectProxy that is used by the LiveWidgets to register their own topic callbacks. The Moksha Live Socket then handles subscribing widgets to their appropriate topics, decoding the incoming JSON data, and dispatching messages to them as they arrive. """ environ['paste.registry'].register(moksha.livewidgets, { 'onopen': [], 'onclose': [], 'onerror': [], 'onerrorframe': [], 'onconnectedframe': [], 'onmessageframe': defaultdict(list) # {topic: [js_callback,]} })
def register_livewidgets(self, environ): """ Register the `moksha.livewidgets` dictionary. This is a per-request StackedObjectProxy that is used by the LiveWidgets to register their own topic callbacks. The Moksha Live Socket then handles subscribing widgets to their appropriate topics, decoding the incoming JSON data, and dispatching messages to them as they arrive. """ environ['paste.registry'].register( moksha.utils.livewidgets, { 'onopen': [], 'onclose': [], 'onerror': [], 'onerrorframe': [], 'onconnectedframe': [], 'onmessageframe': defaultdict(list) # {topic: [js_callback,]} })
def update_params(self, d): super(AMQPSocket, self).update_params(d) d.topics = [] d.onmessageframe = defaultdict(str) # {topic: 'js callbacks'} for callback in self.callbacks: if len(moksha.livewidgets[callback]): cbs = '' if callback == 'onmessageframe': for topic in moksha.livewidgets[callback]: d.topics.append(topic) for cb in moksha.livewidgets[callback][topic]: d.onmessageframe[topic] += '%s;' % str(cb) else: for cb in moksha.livewidgets[callback]: if isinstance(cb, (js_callback, js_function)): cbs += '$(%s);' % str(cb) else: cbs += str(cb) if cbs: d[callback] = cbs
def __init__(self, topics=None): global config self.amqp_broker = config.get('amqp_broker', None) self.stomp_broker = config.get('stomp_broker', None) # If we're running outside of middleware and hub, load config if not self.amqp_broker and not self.stomp_broker: config = get_moksha_appconfig() self.amqp_broker = config.get('amqp_broker', None) self.stomp_broker = config.get('stomp_broker', None) if self.amqp_broker and self.stomp_broker: log.warning("Running with both a STOMP and AMQP broker. " "This mode is experimental and may or may not work") if not self.topics: self.topics = defaultdict(list) if topics: for topic, callbacks in topics.iteritems(): if not isinstance(callbacks, list): callbacks = [callbacks] for callback in callbacks: self.topics[topic].append(callback) if self.amqp_broker: AMQPHub.__init__(self, self.amqp_broker, username=config.get('amqp_broker_user', 'guest'), password=config.get('amqp_broker_pass', 'guest'), ssl=config.get('amqp_broker_ssl', False)) if self.stomp_broker: log.info('Initializing STOMP support') StompHub.__init__(self, self.stomp_broker, port=config.get('stomp_port', 61613), username=config.get('stomp_user', 'guest'), password=config.get('stomp_pass', 'guest'), topics=self.topics.keys())
def prepare(self): super(AMQPSocket, self).prepare() self.orbited_url = '%s://%s:%s' % ( self.orbited_scheme, self.orbited_host, self.orbited_port) self.topics = [] self.onmessageframe = defaultdict(str) # {topic: 'js callbacks'} for callback in self.callbacks: if len(moksha.utils.livewidgets[callback]): cbs = '' if callback == 'onmessageframe': for topic in moksha.utils.livewidgets[callback]: self.topics.append(topic) for cb in moksha.utils.livewidgets[callback][topic]: self.onmessageframe[topic] += '%s;' % str(cb) else: for cb in moksha.utils.livewidgets[callback]: if isinstance(cb, (twc.js_callback, twc.js_function)): cbs += '$(%s);' % str(cb) else: cbs += str(cb) if cbs: setattr(self, callback, cbs)
def prepare(self): if not self.orbited_url: self.orbited_url = '%s://%s:%s' % ( self.orbited_scheme, self.orbited_host, self.orbited_port) super(StompWidget, self).prepare() self.onmessageframe = defaultdict(str) # {topic: 'js callbacks'} for callback in self.callbacks: if len(moksha.utils.livewidgets[callback]): cbs = '' if callback == 'onmessageframe': for topic in moksha.utils.livewidgets[callback]: self.topics.append(topic) for cb in moksha.utils.livewidgets[callback][topic]: self.onmessageframe[topic] += '%s;' % str(cb) else: for cb in moksha.utils.livewidgets[callback]: if isinstance(cb, (twc.js_callback, js_function)): cbs += '$(%s);' % str(cb) else: cbs += str(cb) if cbs: setattr(self, callback, cbs) if self.notify: moksha_notify.register_resources() openmsg = "Moksha live socket connected" self.onopen = twc.js_callback('function() { $.jGrowl("%s") }' % openmsg) errormsg = "Moksha Live Socket Error: " self.onerror = twc.js_callback( 'function(error) { $.jGrowl("%s" + error) }' % errormsg) errorframemsg = "Error frame received from Moksha Socket: " self.onerrorframe = twc.js_callback( 'function(f) { $.jGrowl("%s" + f) }' % errorframemsg) closemsg = "Moksha Socket Closed" self.onclose = twc.js_callback('function(c) { $.jGrowl("%s") }' % closemsg)
def poll(self): i = 0 mem_data = { 'data': [], 'options': { 'xaxis': { 'ticks': [] }, 'legend': { 'position': 'nw', 'backgroundColor': 'null' } } } cpu_data = { 'data': [], 'options': { 'xaxis': { 'min': 0, 'max': 50 }, 'yaxis': { 'min': 0, 'max': 100 * self.processors }, 'legend': { 'position': 'nw', 'backgroundColor': 'null' } } } self.count += 1 if self.poll_for_new_pids and self.count % 10 == 0: self.programs = self._find_programs() self.count = 0 for program in self.programs: for proc in self.procs: if program[NAME].startswith(proc) or proc in program[NAME]: total_mem_usage = float(program[MEM_TOTAL].split()[0]) mem_data['data'].append({ 'data': [[i, total_mem_usage]], 'bars': { 'show': 'true' }, 'label': program[NAME], }) mem_data['options']['xaxis']['ticks'].append( [i + 0.5, program[NAME]]) i += 1 self.send_message('moksha_mem_metrics', [mem_data]) # top only allows 20 pids to be specified at a time # so make multiple calls if we have to out = [] _pids = [] _pids.extend(self.pids.keys()) while _pids: cmd = ['/usr/bin/top', '-b', '-n 1'] for pid in _pids[:20]: cmd += ['-p %s' % pid] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = p.communicate() stdout = stdout.strip().split('\n') for i, line in enumerate(stdout): if line.lstrip().startswith('PID'): out += stdout[i + 1:] break _pids = _pids[20:] # Determine the % CPU usage for each pid pid_cpu = defaultdict(float) for line in out: splitline = line.split() if not splitline: continue pid = int(splitline[0]) cpu_usage = float(splitline[-4]) pid_cpu[self.pids[pid]] += cpu_usage # Move everything to the left, and append the latest CPU usage. for program, cpu_usage in pid_cpu.items(): for history in self.cpu_usage[program]: history[0] -= 1 self.cpu_usage[program].append([50, cpu_usage]) # Only store 50 ticks worth of data self.cpu_usage[program] = self.cpu_usage[program][-51:] for program, history in self.cpu_usage.items(): cpu_data['data'].append({ 'data': history, 'lines': { 'show': 'true', 'fill': 'true' }, 'label': program.split()[0], }) self.send_message('moksha_cpu_metrics', [cpu_data])
class MokshaMetricsDataStream(PollingDataStream): frequency = 3 procs = ('orbited', 'paster', 'moksha', 'httpd') cpu_usage = defaultdict(list) programs = None pids = {} count = 0 # Unless we are monitoring apache, only poll for pids once poll_for_new_pids = False def __init__(self): self.programs = self._find_programs() self.processors = self._find_processors() super(MokshaMetricsDataStream, self).__init__() def _find_programs(self): programs = [] for program in self.mem(): for proc in self.procs: if program[NAME].startswith(proc) or proc in program[NAME]: programs.append(program) for pid in program[PID].split(','): self.pids[int(pid)] = program[NAME] if proc == 'httpd': self.poll_for_new_pids = True return programs def _find_processors(self): """ Find the number of processors """ processors = 0 for line in file('/proc/cpuinfo').readlines(): if line.startswith('processor'): processors = int(line.split(':')[1]) return processors + 1 def poll(self): i = 0 mem_data = { 'data': [], 'options': { 'xaxis': { 'ticks': [] }, 'legend': { 'position': 'nw', 'backgroundColor': 'null' } } } cpu_data = { 'data': [], 'options': { 'xaxis': { 'min': 0, 'max': 50 }, 'yaxis': { 'min': 0, 'max': 100 * self.processors }, 'legend': { 'position': 'nw', 'backgroundColor': 'null' } } } self.count += 1 if self.poll_for_new_pids and self.count % 10 == 0: self.programs = self._find_programs() self.count = 0 for program in self.programs: for proc in self.procs: if program[NAME].startswith(proc) or proc in program[NAME]: total_mem_usage = float(program[MEM_TOTAL].split()[0]) mem_data['data'].append({ 'data': [[i, total_mem_usage]], 'bars': { 'show': 'true' }, 'label': program[NAME], }) mem_data['options']['xaxis']['ticks'].append( [i + 0.5, program[NAME]]) i += 1 self.send_message('moksha_mem_metrics', [mem_data]) # top only allows 20 pids to be specified at a time # so make multiple calls if we have to out = [] _pids = [] _pids.extend(self.pids.keys()) while _pids: cmd = ['/usr/bin/top', '-b', '-n 1'] for pid in _pids[:20]: cmd += ['-p %s' % pid] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, stderr = p.communicate() stdout = stdout.strip().split('\n') for i, line in enumerate(stdout): if line.lstrip().startswith('PID'): out += stdout[i + 1:] break _pids = _pids[20:] # Determine the % CPU usage for each pid pid_cpu = defaultdict(float) for line in out: splitline = line.split() if not splitline: continue pid = int(splitline[0]) cpu_usage = float(splitline[-4]) pid_cpu[self.pids[pid]] += cpu_usage # Move everything to the left, and append the latest CPU usage. for program, cpu_usage in pid_cpu.items(): for history in self.cpu_usage[program]: history[0] -= 1 self.cpu_usage[program].append([50, cpu_usage]) # Only store 50 ticks worth of data self.cpu_usage[program] = self.cpu_usage[program][-51:] for program, history in self.cpu_usage.items(): cpu_data['data'].append({ 'data': history, 'lines': { 'show': 'true', 'fill': 'true' }, 'label': program.split()[0], }) self.send_message('moksha_cpu_metrics', [cpu_data]) def mem(self): """ Returns a list of per-program memory usage. Private + Shared = RAM used Program [["39.4 MiB", "10.3 MiB", "49.8 MiB", "Xorg"], ["42.2 MiB", "12.4 MiB", "54.6 MiB", "nautilus"], ["52.3 MiB", "10.8 MiB", "63.0 MiB", "liferea-bin"] ["171.6 MiB", "11.9 MiB", "183.5 MiB", "firefox-bin"]] Taken from the ps_mem.py script written by Pádraig Brady. http://www.pixelbeat.org/scripts/ps_mem.py """ our_pid = os.getpid() results = [] global have_pss have_pss = 0 def kernel_ver(): """ (major,minor,release) """ kv = open("/proc/sys/kernel/osrelease").readline().split(".")[:3] for char in "-_": kv[2] = kv[2].split(char)[0] return (int(kv[0]), int(kv[1]), int(kv[2])) kv = kernel_ver() def getMemStats(pid): """ return Private,Shared """ global have_pss Private_lines = [] Shared_lines = [] Pss_lines = [] pagesize = os.sysconf("SC_PAGE_SIZE") / 1024 #KiB Rss = int( open("/proc/" + str(pid) + "/statm").readline().split()[1]) * pagesize if os.path.exists("/proc/" + str(pid) + "/smaps"): #stat for line in open("/proc/" + str(pid) + "/smaps").readlines(): #open if line.startswith("Shared"): Shared_lines.append(line) elif line.startswith("Private"): Private_lines.append(line) elif line.startswith("Pss"): have_pss = 1 Pss_lines.append(line) Shared = sum([int(line.split()[1]) for line in Shared_lines]) Private = sum([int(line.split()[1]) for line in Private_lines]) #Note Shared + Private = Rss above #The Rss in smaps includes video card mem etc. if have_pss: pss_adjust = 0.5 #add 0.5KiB as this average error due to trunctation Pss = sum([ float(line.split()[1]) + pss_adjust for line in Pss_lines ]) Shared = Pss - Private elif (2, 6, 1) <= kv <= (2, 6, 9): Shared = 0 #lots of overestimation, but what can we do? Private = Rss else: Shared = int( open("/proc/" + str(pid) + "/statm").readline().split()[2]) * pagesize Private = Rss - Shared return (Private, Shared) def getCmdName(pid): cmd = file("/proc/%d/status" % pid).readline()[6:-1] exe = os.path.basename(os.path.realpath("/proc/%d/exe" % pid)) if exe.startswith(cmd): cmd = exe #show non truncated version #Note because we show the non truncated name #one can have separated programs as follows: #584.0 KiB + 1.0 MiB = 1.6 MiB mozilla-thunder (exe -> bash) # 56.0 MiB + 22.2 MiB = 78.2 MiB mozilla-thunderbird-bin return cmd cmds = {} shareds = {} count = {} pids = {} for pid in os.listdir("/proc/"): try: pid = int( pid) #note Thread IDs not listed in /proc/ which is good #if pid == our_pid: continue except: continue try: cmd = getCmdName(pid) except Exception, e: #permission denied or #kernel threads don't have exe links or #process gone continue try: private, shared = getMemStats(pid) except: continue #process gone if shareds.get(cmd): if have_pss: #add shared portion of PSS together shareds[cmd] += shared elif shareds[cmd] < shared: #just take largest shared val shareds[cmd] = shared else: shareds[cmd] = shared cmds[cmd] = cmds.setdefault(cmd, 0) + private if count.has_key(cmd): count[cmd] += 1 pids[cmd] += ',%d' % pid else: count[cmd] = 1 pids[cmd] = str(pid) #Add shared mem for each program total = 0 for cmd in cmds.keys(): cmds[cmd] = cmds[cmd] + shareds[cmd] total += cmds[cmd] #valid if PSS available sort_list = cmds.items() sort_list.sort(lambda x, y: cmp(x[1], y[1])) sort_list = filter(lambda x: x[1], sort_list) #get rid of zero sized processes #The following matches "du -h" output def human(num, power="Ki"): powers = ["Ki", "Mi", "Gi", "Ti"] while num >= 1000: #4 digits num /= 1024.0 power = powers[powers.index(power) + 1] return "%.1f %s" % (num, power) def cmd_with_count(cmd, count): if count > 1: return "%s (%u)" % (cmd, count) else: return cmd for cmd in sort_list: results.append([ "%s" % pids[cmd[0]], "%sB" % human(cmd[1] - shareds[cmd[0]]), "%sB" % human(shareds[cmd[0]]), "%sB" % human(cmd[1]), "%s" % cmd_with_count(cmd[0], count[cmd[0]]) ]) if have_pss: results.append(["", "", "", "%sB" % human(total)]) return results
def poll(self): i = 0 mem_data = { 'data': [], 'options': { 'xaxis': {'ticks': []}, 'legend': {'position': 'nw', 'backgroundColor': 'null'} } } cpu_data = { 'data': [], 'options': { 'xaxis': {'min': 0, 'max': 50}, 'yaxis': {'min': 0, 'max': 100 * self.processors}, 'legend': { 'position': 'nw', 'backgroundColor': 'null' } } } self.count += 1 if self.poll_for_new_pids and self.count % 10 == 0: self.programs = self._find_programs() self.count = 0 for program in self.programs: total_mem_usage = float(program[MEM_TOTAL].split()[0]) mem_data['data'].append({ 'data': [[i, total_mem_usage]], 'bars': {'show': 'true'}, 'label': program[NAME], }) mem_data['options']['xaxis']['ticks'].append( [i + 0.5, program[NAME]]) i += 1 self.send_message('moksha_mem_metrics', [mem_data]) # top only allows 20 pids to be specified at a time # so make multiple calls if we have to out = [] _pids = [] _pids.extend(self.pids.keys()) while _pids: cmd = ['/usr/bin/top', '-b', '-n 1'] for pid in _pids[:20]: cmd += ['-p %s' % pid] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) try: stdout, stderr = p.communicate() except IOError: self.log.warning("Unable to communicate with `top` subprocess") return stdout = stdout.strip().split('\n') for i, line in enumerate(stdout): if line.lstrip().startswith('PID'): out += stdout[i+1:] break _pids = _pids[20:] # Determine the % CPU usage for each pid pid_cpu = defaultdict(float) for line in out: splitline = line.split() if not splitline: continue pid = int(splitline[0]) cpu_usage = float(splitline[-4]) pid_cpu[self.pids[pid]] += cpu_usage # Move everything to the left, and append the latest CPU usage. for program, cpu_usage in pid_cpu.items(): program = program.split()[0] for history in self.cpu_usage[program]: history[0] -= 1 self.cpu_usage[program].append([50, cpu_usage]) # Only store 50 ticks worth of data self.cpu_usage[program] = self.cpu_usage[program][-51:] for proc in self.procs: for program, history in self.cpu_usage.items(): if program.startswith(proc) or proc in program: cpu_data['data'].append({ 'data': history, 'lines': {'show': 'true', 'fill': 'true'}, 'label': program, }) self.send_message('moksha_cpu_metrics', [cpu_data])