class DataStore: '''The Datastore object stores all of the metric data as well as summary data for all data sources. It is a singleton object which guarantees that all instances of the object will produce the same data store. ''' _shared_state = {} #Storage for a singleton object _initialized = False lock = thread.allocate_lock() def __init__(self): # Replace the objects attributes with the original # attributes and data. self.__dict__ = DataStore._shared_state # Make sure that the DataStore object is only initialized once. if not DataStore._initialized: # Allocate a lock that will be used by all threads # that are reading or writing to the data store. self.lock = thread.allocate_lock() self.lock.acquire() self.rootElement = None # Initialize the data store with the GANGLIA_XML and GRID tags. # Data store should never be completely empty. Even if there are # no reporting data sources the web front end depends on having # at least a GANGLIA_XML tag and a nested GRID tag. cfg = getConfig() self.setNode( Element('GANGLIA_XML', { 'VERSION': cfg.VERSION, 'SOURCE': 'gmetad' })) self.setNode( Element( 'GRID', { 'NAME': cfg[GmetadConfig.GRIDNAME], 'AUTHORITY': cfg[GmetadConfig.AUTHORITY], 'LOCALTIME': '%d' % time.time() }), self.rootElement) self.lock.release() # Start up the grid summary thread. self.gridSummary = DataStoreGridSummary() self.gridSummary.start() # Start up the plugin notifier. self.notifier = GmetadNotifier() self.notifier.start() DataStore._initialized = True def _doGridSummary(self, gridNode): '''This private function will calculate the summaries for a single grid. It is called each time new data has been read for a specific grid. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. gridNode.summaryData = {} gridNode.summaryData['summary'] = {} gridNode.summaryData['hosts_up'] = 0 gridNode.summaryData['hosts_down'] = 0 gridUp = (gridNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for clusterNode in gridNode: # Assume that cluster is up if we get a clusterNode object clusterUp = True for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax')) * 4): gridNode.summaryData['hosts_up'] += 1 else: gridNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = gridNode.summaryData['summary'][str( metricNode)] currSum = summaryNode.getAttr('sum') summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') # Add the summary node to the summary dictionary gridNode.summaryData['summary'][str( summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self) def _doClusterSummary(self, clusterNode): '''This private function will calculate the summaries for a single cluster. It is called each time that new data has been read for a specific cluster. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. clusterNode.summaryData = {} clusterNode.summaryData['summary'] = {} clusterNode.summaryData['hosts_up'] = 0 clusterNode.summaryData['hosts_down'] = 0 clusterUp = (clusterNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax')) * 4): clusterNode.summaryData['hosts_up'] += 1 else: clusterNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if metricNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = clusterNode.summaryData['summary'][str( metricNode)] currSum = summaryNode.getAttr('sum') summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') # Add the summary node to the summary dictionary clusterNode.summaryData['summary'][str( summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self) def shutdown(self): # Shut down the notifier and the grid summary threads self.notifier.shutdown() self.gridSummary.shutdown() def setNode(self, node, parent=None): ''' Add a new node to the data store in the appropriate position in the tree. ''' if parent is None: # If there isn't a root node, the new node becomes the root if self.rootElement is None: self.rootElement = node return self.rootElement if str(node) in parent.children: try: node.children = parent[str(node)].children except AttributeError: pass try: node.summaryData = parent[str(node)].summaryData except AttributeError: pass # Add the new node as a child of the parent parent[str(node)] = node return parent[str(node)] def getNode(self, ancestry=[]): ''' Find a node in the data store based on a node path ''' # If no path was given, just return the root node. if not len(ancestry): return self.rootElement node = None # Follow the path given in the ancestry list until the # correct node is found. while ancestry: nodeId = ancestry.pop(0) if node is None: if nodeId == str(self.rootElement): node = self.rootElement else: return None else: try: node = node[nodeId] except KeyError: pass return node def updateFinished(self, clusterNode): ''' This method is called when the gmond reader has finished updating a cluster. It indicates that a summary can be done over the entire cluster and then the cluster transaction needs to be entered and passed to the plugins. ''' if clusterNode is not None: if 'CLUSTER' == clusterNode.id: self._doClusterSummary(clusterNode) if 'GRID' == clusterNode.id: self._doGridSummary(clusterNode) self.notifier.insertTransaction(clusterNode) def acquireLock(self, obj): ''' Acquire a data store lock. ''' self.lock.acquire() logging.debug('DataStore lock acquired %s' % str(obj)) def releaseLock(self, obj): ''' Release the data store lock. ''' self.lock.release() logging.debug('DataStore lock released%s' % str(obj))
class DataStore: '''The Datastore object stores all of the metric data as well as summary data for all data sources. It is a singleton object which guarantees that all instances of the object will produce the same data store. ''' _shared_state = {} #Storage for a singleton object _initialized = False lock = thread.allocate_lock() def __init__(self): # Replace the objects attributes with the original # attributes and data. self.__dict__ = DataStore._shared_state # Make sure that the DataStore object is only initialized once. if not DataStore._initialized: # Allocate a lock that will be used by all threads # that are reading or writing to the data store. self.lock = thread.allocate_lock() self.lock.acquire() self.rootElement = None # Initialize the data store with the GANGLIA_XML and GRID tags. # Data store should never be completely empty. Even if there are # no reporting data sources the web front end depends on having # at least a GANGLIA_XML tag and a nested GRID tag. cfg = getConfig() self.setNode(Element('GANGLIA_XML', {'VERSION':cfg.VERSION, 'SOURCE':'gmetad'})) self.setNode(Element('GRID', {'NAME':cfg[GmetadConfig.GRIDNAME], 'AUTHORITY':cfg[GmetadConfig.AUTHORITY], 'LOCALTIME':'%d' % time.time()}), self.rootElement) self.lock.release() # Start up the grid summary thread. self.gridSummary = DataStoreGridSummary() self.gridSummary.start() # Start up the plugin notifier. self.notifier = GmetadNotifier() self.notifier.start() DataStore._initialized = True def _doGridSummary(self, gridNode): '''This private function will calculate the summaries for a single grid. It is called each time new data has been read for a specific grid. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. gridNode.summaryData = {} gridNode.summaryData['summary'] = {} gridNode.summaryData['hosts_up'] = 0 gridNode.summaryData['hosts_down'] = 0 gridUp = (gridNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for clusterNode in gridNode: # Assume that cluster is up if we get a clusterNode object clusterUp = True for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4): gridNode.summaryData['hosts_up'] += 1 else: gridNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = gridNode.summaryData['summary'][str(metricNode)] currSum = summaryNode.getAttr('sum') summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') # Add the summary node to the summary dictionary gridNode.summaryData['summary'][str(summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self) def _doClusterSummary(self, clusterNode): '''This private function will calculate the summaries for a single cluster. It is called each time that new data has been read for a specific cluster. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. clusterNode.summaryData = {} clusterNode.summaryData['summary'] = {} clusterNode.summaryData['hosts_up'] = 0 clusterNode.summaryData['hosts_down'] = 0 clusterUp = (clusterNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4): clusterNode.summaryData['hosts_up'] += 1 else: clusterNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if metricNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = clusterNode.summaryData['summary'][str(metricNode)] currSum = summaryNode.getAttr('sum') summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') # Add the summary node to the summary dictionary clusterNode.summaryData['summary'][str(summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self) def shutdown(self): # Shut down the notifier and the grid summary threads self.notifier.shutdown() self.gridSummary.shutdown() def setNode(self, node, parent=None): ''' Add a new node to the data store in the appropriate position in the tree. ''' if parent is None: # If there isn't a root node, the new node becomes the root if self.rootElement is None: self.rootElement = node return self.rootElement if str(node) in parent.children: try: node.children = parent[str(node)].children except AttributeError: pass try: node.summaryData = parent[str(node)].summaryData except AttributeError: pass # Add the new node as a child of the parent parent[str(node)] = node return parent[str(node)] def getNode(self, ancestry=[]): ''' Find a node in the data store based on a node path ''' # If no path was given, just return the root node. if not len(ancestry): return self.rootElement node = None # Follow the path given in the ancestry list until the # correct node is found. while ancestry: nodeId = ancestry.pop(0) if node is None: if nodeId == str(self.rootElement): node = self.rootElement else: return None else: try: node = node[nodeId] except KeyError: pass return node def updateFinished(self, clusterNode): ''' This method is called when the gmond reader has finished updating a cluster. It indicates that a summary can be done over the entire cluster and then the cluster transaction needs to be entered and passed to the plugins. ''' if clusterNode is not None: if 'CLUSTER' == clusterNode.id: self._doClusterSummary(clusterNode); if 'GRID' == clusterNode.id: self._doGridSummary(clusterNode) self.notifier.insertTransaction(clusterNode) def acquireLock(self, obj): ''' Acquire a data store lock. ''' self.lock.acquire() logging.debug('DataStore lock acquired %s'%str(obj)) def releaseLock(self, obj): ''' Release the data store lock. ''' self.lock.release() logging.debug('DataStore lock released%s'%str(obj))
class DataStore: '''The Datastore object stores all of the metric data as well as summary data for all data sources. It is a singleton object which guarantees that all instances of the object will produce the same data store. ''' _shared_state = {} #Storage for a singleton object _initialized = False lock = thread.allocate_lock() def __init__(self): # Replace the objects attributes with the original # attributes and data. self.__dict__ = DataStore._shared_state # Make sure that the DataStore object is only initialized once. if not DataStore._initialized: # Allocate a lock that will be used by all threads # that are reading or writing to the data store. self.lock = thread.allocate_lock() self.lock.acquire() self.rootElement = None # Initialize the data store with the GANGLIA_XML and GRID tags. # Data store should never be completely empty. Even if there are # no reporting data sources the web front end depends on having # at least a GANGLIA_XML tag and a nested GRID tag. cfg = getConfig() self.setNode(Element('GANGLIA_XML', {'VERSION':cfg.VERSION, 'SOURCE':'gmetad'})) self.setNode(Element('GRID', {'NAME':cfg[GmetadConfig.GRIDNAME], 'AUTHORITY':cfg[GmetadConfig.AUTHORITY], 'LOCALTIME':'%d' % time.time()}), self.rootElement) self.lock.release() # Start up the grid summary thread. self.gridSummary = DataStoreGridSummary() self.gridSummary.start() # Start up the plugin notifier. self.notifier = GmetadNotifier() self.notifier.start() DataStore._initialized = True def _doGridSummary(self, gridNode): '''This private function will calculate the summaries for a single grid. It is called each time new data has been read for a specific grid. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. gridNode.summaryData = {} gridNode.summaryData['summary'] = {} gridNode.summaryData['hosts_up'] = 0 gridNode.summaryData['hosts_down'] = 0 gridUp = (gridNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for clusterNode in gridNode: # Assume that cluster is up if we get a clusterNode object clusterUp = True for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4): gridNode.summaryData['hosts_up'] += 1 else: gridNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = gridNode.summaryData['summary'][str(metricNode)] currSum = summaryNode.getAttr('sum') summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') # Add the summary node to the summary dictionary gridNode.summaryData['summary'][str(summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self) def _doClusterSummary(self, clusterNode): '''This private function will calculate the summaries for a single cluster. It is called each time that new data has been read for a specific cluster. All summary data is placed in the summaryData dictionary. ''' self.acquireLock(self) # Clear out the summaryData from the last run and initialize # the dictionary for new summaries. clusterNode.summaryData = {} clusterNode.summaryData['summary'] = {} clusterNode.summaryData['hosts_up'] = 0 clusterNode.summaryData['hosts_down'] = 0 clusterUp = (clusterNode.getAttr('status') == 'up') summaryTime = int(time.time()) # Summarize over each host contained by the cluster for hostNode in clusterNode: reportedTime = summaryTime # Sum up the status of all of the hosts if 'HOST' == hostNode.id: # Calculate the difference between the last known reported time # and the current time. This determines if the host is up or down # ** There may still be some issues with the way that this calculation is done # ** The metric node below may also have the same issues. reportedTime = int(hostNode.getAttr('reported')) tn = int(hostNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if hostNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 hostNode.setAttr('tn', str(tn)) else: hostNode.lastReportedTime = reportedTime try: if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4): clusterNode.summaryData['hosts_up'] += 1 else: clusterNode.summaryData['hosts_down'] += 1 except AttributeError: pass except KeyError: pass # Summarize over each metric within a host for metricNode in hostNode: tn = int(metricNode.getAttr('tn')) # If the last reported time is the same as the current reported time, then # the host has not been updated. Therefore calculate the time offset from # the current time. if metricNode.lastReportedTime == reportedTime: tn = summaryTime - reportedTime if tn < 0: tn = 0 metricNode.setAttr('tn', str(tn)) # Don't include metrics that can not be summarized if metricNode.getAttr('type') in ['string', 'timestamp']: continue try: # Pull the existing summary node from the summary data # dictionary. If one doesn't exist, add it in the exception. summaryNode = clusterNode.summaryData['summary'][str(metricNode)] currSum = summaryNode.getAttr('sum') try : summaryNode.incAttr('sum', float(metricNode.getAttr('val'))) except ValueError, msg: print str(msg) + " sum: " + metricNode.getAttr('val') except KeyError: # Since summary metrics use a different tag, create the new # summary node with correct tag. summaryNode = metricNode.summaryCopy(tag='METRICS') # Initialize the first summary value and change the data type # to double for all metric summaries try : summaryNode.setAttr('sum', float(metricNode.getAttr('val'))) summaryNode.setAttr('type', 'double') except ValueError, msg: print str(msg) + " sum: " + metricNode.getAttr('val') # Add the summary node to the summary dictionary clusterNode.summaryData['summary'][str(summaryNode)] = summaryNode summaryNode.incAttr('num', 1) self.releaseLock(self)