Beispiel #1
0
class DataStore:
    '''The Datastore object stores all of the metric data
       as well as summary data for all data sources. It is
       a singleton object which guarantees that all instances
       of the object will produce the same data store. '''
    _shared_state = {}  #Storage for a singleton object
    _initialized = False
    lock = thread.allocate_lock()

    def __init__(self):
        # Replace the objects attributes with the original
        #  attributes and data.
        self.__dict__ = DataStore._shared_state

        # Make sure that the DataStore object is only initialized once.
        if not DataStore._initialized:
            # Allocate a lock that will be used by all threads
            #  that are reading or writing to the data store.
            self.lock = thread.allocate_lock()
            self.lock.acquire()
            self.rootElement = None
            # Initialize the data store with the GANGLIA_XML and GRID tags.
            # Data store should never be completely empty.  Even if there are
            # no reporting data sources the web front end depends on having
            # at least a GANGLIA_XML tag and a nested GRID tag.
            cfg = getConfig()
            self.setNode(
                Element('GANGLIA_XML', {
                    'VERSION': cfg.VERSION,
                    'SOURCE': 'gmetad'
                }))
            self.setNode(
                Element(
                    'GRID', {
                        'NAME': cfg[GmetadConfig.GRIDNAME],
                        'AUTHORITY': cfg[GmetadConfig.AUTHORITY],
                        'LOCALTIME': '%d' % time.time()
                    }), self.rootElement)
            self.lock.release()

            # Start up the grid summary thread.
            self.gridSummary = DataStoreGridSummary()
            self.gridSummary.start()

            # Start up the plugin notifier.
            self.notifier = GmetadNotifier()
            self.notifier.start()
            DataStore._initialized = True

    def _doGridSummary(self, gridNode):
        '''This private function will calculate the summaries for a
            single grid.  It is called each time new data
            has been read for a specific grid.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        gridNode.summaryData = {}
        gridNode.summaryData['summary'] = {}
        gridNode.summaryData['hosts_up'] = 0
        gridNode.summaryData['hosts_down'] = 0
        gridUp = (gridNode.getAttr('status') == 'up')
        summaryTime = int(time.time())

        # Summarize over each host contained by the cluster
        for clusterNode in gridNode:
            # Assume that cluster is up if we get a clusterNode object
            clusterUp = True
            for hostNode in clusterNode:
                reportedTime = summaryTime
                # Sum up the status of all of the hosts
                if 'HOST' == hostNode.id:
                    # Calculate the difference between the last known reported time
                    #  and the current time.  This determines if the host is up or down
                    # ** There may still be some issues with the way that this calculation is done
                    # ** The metric node below may also have the same issues.
                    reportedTime = int(hostNode.getAttr('reported'))
                    tn = int(hostNode.getAttr('tn'))
                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        hostNode.setAttr('tn', str(tn))
                    else:
                        hostNode.lastReportedTime = reportedTime

                    try:
                        if clusterUp and (int(hostNode.getAttr('tn')) <
                                          int(hostNode.getAttr('tmax')) * 4):
                            gridNode.summaryData['hosts_up'] += 1
                        else:
                            gridNode.summaryData['hosts_down'] += 1
                    except AttributeError:
                        pass
                    except KeyError:
                        pass
                # Summarize over each metric within a host
                for metricNode in hostNode:
                    tn = int(metricNode.getAttr('tn'))

                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        metricNode.setAttr('tn', str(tn))

                    # Don't include metrics that can not be summarized
                    if metricNode.getAttr('type') in ['string', 'timestamp']:
                        continue
                    try:
                        # Pull the existing summary node from the summary data
                        # dictionary. If one doesn't exist, add it in the exception.
                        summaryNode = gridNode.summaryData['summary'][str(
                            metricNode)]
                        currSum = summaryNode.getAttr('sum')
                        summaryNode.incAttr('sum',
                                            float(metricNode.getAttr('val')))
                    except KeyError:
                        # Since summary metrics use a different tag, create the new
                        #  summary node with correct tag.
                        summaryNode = metricNode.summaryCopy(tag='METRICS')
                        # Initialize the first summary value and change the data type
                        # to double for all metric summaries
                        summaryNode.setAttr('sum',
                                            float(metricNode.getAttr('val')))
                        summaryNode.setAttr('type', 'double')
                        # Add the summary node to the summary dictionary
                        gridNode.summaryData['summary'][str(
                            summaryNode)] = summaryNode
                    summaryNode.incAttr('num', 1)
        self.releaseLock(self)

    def _doClusterSummary(self, clusterNode):
        '''This private function will calculate the summaries for a
            single cluster.  It is called each time that new data
            has been read for a specific cluster.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        clusterNode.summaryData = {}
        clusterNode.summaryData['summary'] = {}
        clusterNode.summaryData['hosts_up'] = 0
        clusterNode.summaryData['hosts_down'] = 0
        clusterUp = (clusterNode.getAttr('status') == 'up')
        summaryTime = int(time.time())

        # Summarize over each host contained by the cluster
        for hostNode in clusterNode:
            reportedTime = summaryTime
            # Sum up the status of all of the hosts
            if 'HOST' == hostNode.id:
                # Calculate the difference between the last known reported time
                #  and the current time.  This determines if the host is up or down
                # ** There may still be some issues with the way that this calculation is done
                # ** The metric node below may also have the same issues.
                reportedTime = int(hostNode.getAttr('reported'))
                tn = int(hostNode.getAttr('tn'))
                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if hostNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    hostNode.setAttr('tn', str(tn))
                else:
                    hostNode.lastReportedTime = reportedTime

                try:
                    if clusterUp and (int(hostNode.getAttr('tn')) <
                                      int(hostNode.getAttr('tmax')) * 4):
                        clusterNode.summaryData['hosts_up'] += 1
                    else:
                        clusterNode.summaryData['hosts_down'] += 1
                except AttributeError:
                    pass
                except KeyError:
                    pass
            # Summarize over each metric within a host
            for metricNode in hostNode:
                tn = int(metricNode.getAttr('tn'))

                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if metricNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    metricNode.setAttr('tn', str(tn))

                # Don't include metrics that can not be summarized
                if metricNode.getAttr('type') in ['string', 'timestamp']:
                    continue
                try:
                    # Pull the existing summary node from the summary data
                    # dictionary. If one doesn't exist, add it in the exception.
                    summaryNode = clusterNode.summaryData['summary'][str(
                        metricNode)]
                    currSum = summaryNode.getAttr('sum')
                    summaryNode.incAttr('sum',
                                        float(metricNode.getAttr('val')))
                except KeyError:
                    # Since summary metrics use a different tag, create the new
                    #  summary node with correct tag.
                    summaryNode = metricNode.summaryCopy(tag='METRICS')
                    # Initialize the first summary value and change the data type
                    # to double for all metric summaries
                    summaryNode.setAttr('sum',
                                        float(metricNode.getAttr('val')))
                    summaryNode.setAttr('type', 'double')
                    # Add the summary node to the summary dictionary
                    clusterNode.summaryData['summary'][str(
                        summaryNode)] = summaryNode
                summaryNode.incAttr('num', 1)
        self.releaseLock(self)

    def shutdown(self):
        # Shut down the notifier and the grid summary threads
        self.notifier.shutdown()
        self.gridSummary.shutdown()

    def setNode(self, node, parent=None):
        ''' Add a new node to the data store in the appropriate
            position in the tree. '''
        if parent is None:
            # If there isn't a root node, the new node becomes the root
            if self.rootElement is None:
                self.rootElement = node
            return self.rootElement
        if str(node) in parent.children:
            try:
                node.children = parent[str(node)].children
            except AttributeError:
                pass
            try:
                node.summaryData = parent[str(node)].summaryData
            except AttributeError:
                pass
        # Add the new node as a child of the parent
        parent[str(node)] = node
        return parent[str(node)]

    def getNode(self, ancestry=[]):
        ''' Find a node in the data store based on a node path '''
        # If no path was given, just return the root node.
        if not len(ancestry):
            return self.rootElement
        node = None
        # Follow the path given in the ancestry list until the
        #  correct node is found.
        while ancestry:
            nodeId = ancestry.pop(0)
            if node is None:
                if nodeId == str(self.rootElement):
                    node = self.rootElement
                else:
                    return None
            else:
                try:
                    node = node[nodeId]
                except KeyError:
                    pass
        return node

    def updateFinished(self, clusterNode):
        ''' This method is called when the gmond reader has finished updating
            a cluster.  It indicates that a summary can be done over the
            entire cluster and then the cluster transaction needs to be
            entered and passed to the plugins. '''
        if clusterNode is not None:
            if 'CLUSTER' == clusterNode.id:
                self._doClusterSummary(clusterNode)
            if 'GRID' == clusterNode.id:
                self._doGridSummary(clusterNode)
            self.notifier.insertTransaction(clusterNode)

    def acquireLock(self, obj):
        ''' Acquire a data store lock. '''
        self.lock.acquire()
        logging.debug('DataStore lock acquired %s' % str(obj))

    def releaseLock(self, obj):
        ''' Release the data store lock. '''
        self.lock.release()
        logging.debug('DataStore lock released%s' % str(obj))
Beispiel #2
0
class DataStore:
    '''The Datastore object stores all of the metric data
       as well as summary data for all data sources. It is
       a singleton object which guarantees that all instances
       of the object will produce the same data store. '''
    _shared_state = {} #Storage for a singleton object
    _initialized = False
    lock = thread.allocate_lock()
    
    def __init__(self):
        # Replace the objects attributes with the original
        #  attributes and data.
        self.__dict__ = DataStore._shared_state

        # Make sure that the DataStore object is only initialized once.
        if not DataStore._initialized:
            # Allocate a lock that will be used by all threads
            #  that are reading or writing to the data store.
            self.lock = thread.allocate_lock()
            self.lock.acquire()
            self.rootElement = None
            # Initialize the data store with the GANGLIA_XML and GRID tags.
            # Data store should never be completely empty.  Even if there are
            # no reporting data sources the web front end depends on having
            # at least a GANGLIA_XML tag and a nested GRID tag.
            cfg = getConfig()
            self.setNode(Element('GANGLIA_XML', {'VERSION':cfg.VERSION, 'SOURCE':'gmetad'}))
            self.setNode(Element('GRID', {'NAME':cfg[GmetadConfig.GRIDNAME], 'AUTHORITY':cfg[GmetadConfig.AUTHORITY], 'LOCALTIME':'%d' % time.time()}), self.rootElement)
            self.lock.release()

            # Start up the grid summary thread. 
            self.gridSummary = DataStoreGridSummary()
            self.gridSummary.start()

            # Start up the plugin notifier.
            self.notifier = GmetadNotifier()
            self.notifier.start()
            DataStore._initialized = True

    def _doGridSummary(self, gridNode):
        '''This private function will calculate the summaries for a
            single grid.  It is called each time new data
            has been read for a specific grid.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        gridNode.summaryData = {}
        gridNode.summaryData['summary'] = {}
        gridNode.summaryData['hosts_up'] = 0
        gridNode.summaryData['hosts_down'] = 0
        gridUp = (gridNode.getAttr('status') == 'up')
        summaryTime = int(time.time())

        # Summarize over each host contained by the cluster
        for clusterNode in gridNode:
            # Assume that cluster is up if we get a clusterNode object
            clusterUp = True
            for hostNode in clusterNode:
                reportedTime = summaryTime
                # Sum up the status of all of the hosts
                if 'HOST' == hostNode.id:
                    # Calculate the difference between the last known reported time
                    #  and the current time.  This determines if the host is up or down
                    # ** There may still be some issues with the way that this calculation is done
                    # ** The metric node below may also have the same issues.
                    reportedTime = int(hostNode.getAttr('reported'))
                    tn = int(hostNode.getAttr('tn'))
                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        hostNode.setAttr('tn', str(tn))
                    else:
                        hostNode.lastReportedTime = reportedTime

                    try:
                        if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4):
                            gridNode.summaryData['hosts_up'] += 1
                        else:
                            gridNode.summaryData['hosts_down'] += 1
                    except AttributeError:
                        pass
                    except KeyError:
                        pass
                # Summarize over each metric within a host
                for metricNode in hostNode:
                    tn = int(metricNode.getAttr('tn'))

                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        metricNode.setAttr('tn', str(tn))

                    # Don't include metrics that can not be summarized
                    if metricNode.getAttr('type') in ['string', 'timestamp']:
                        continue
                    try:
                        # Pull the existing summary node from the summary data
                        # dictionary. If one doesn't exist, add it in the exception.
                        summaryNode = gridNode.summaryData['summary'][str(metricNode)]
                        currSum = summaryNode.getAttr('sum')
                        summaryNode.incAttr('sum',  float(metricNode.getAttr('val')))
                    except KeyError:
                        # Since summary metrics use a different tag, create the new 
                        #  summary node with correct tag.
                        summaryNode = metricNode.summaryCopy(tag='METRICS')
                        # Initialize the first summary value and change the data type
                        # to double for all metric summaries
                        summaryNode.setAttr('sum', float(metricNode.getAttr('val')))
                        summaryNode.setAttr('type', 'double')
                        # Add the summary node to the summary dictionary
                        gridNode.summaryData['summary'][str(summaryNode)] = summaryNode
                    summaryNode.incAttr('num', 1)
        self.releaseLock(self)

    def _doClusterSummary(self, clusterNode):
        '''This private function will calculate the summaries for a
            single cluster.  It is called each time that new data
            has been read for a specific cluster.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        clusterNode.summaryData = {}
        clusterNode.summaryData['summary'] = {}
        clusterNode.summaryData['hosts_up'] = 0
        clusterNode.summaryData['hosts_down'] = 0
        clusterUp = (clusterNode.getAttr('status') == 'up')
        summaryTime = int(time.time())
        
        # Summarize over each host contained by the cluster
        for hostNode in clusterNode:
            reportedTime = summaryTime
            # Sum up the status of all of the hosts
            if 'HOST' == hostNode.id:
                # Calculate the difference between the last known reported time
                #  and the current time.  This determines if the host is up or down
                # ** There may still be some issues with the way that this calculation is done
                # ** The metric node below may also have the same issues.
                reportedTime = int(hostNode.getAttr('reported'))
                tn = int(hostNode.getAttr('tn'))
                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if hostNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    hostNode.setAttr('tn', str(tn))
                else:
                    hostNode.lastReportedTime = reportedTime
                    
                try:
                    if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4):
                        clusterNode.summaryData['hosts_up'] += 1
                    else:
                        clusterNode.summaryData['hosts_down'] += 1
                except AttributeError:
                    pass
                except KeyError:
                    pass
            # Summarize over each metric within a host
            for metricNode in hostNode:
                tn = int(metricNode.getAttr('tn'))
                    
                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if metricNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    metricNode.setAttr('tn', str(tn))
                    
                # Don't include metrics that can not be summarized
                if metricNode.getAttr('type') in ['string', 'timestamp']:
                    continue
                try:
                    # Pull the existing summary node from the summary data
                    # dictionary. If one doesn't exist, add it in the exception.
                    summaryNode = clusterNode.summaryData['summary'][str(metricNode)]
                    currSum = summaryNode.getAttr('sum')
                    summaryNode.incAttr('sum',  float(metricNode.getAttr('val')))
                except KeyError:
                    # Since summary metrics use a different tag, create the new 
                    #  summary node with correct tag.
                    summaryNode = metricNode.summaryCopy(tag='METRICS')
                    # Initialize the first summary value and change the data type
                    # to double for all metric summaries
                    summaryNode.setAttr('sum', float(metricNode.getAttr('val')))
                    summaryNode.setAttr('type', 'double')
                    # Add the summary node to the summary dictionary
                    clusterNode.summaryData['summary'][str(summaryNode)] = summaryNode
                summaryNode.incAttr('num', 1)
        self.releaseLock(self)
    
    def shutdown(self):
        # Shut down the notifier and the grid summary threads
        self.notifier.shutdown()
        self.gridSummary.shutdown()
        
    def setNode(self, node, parent=None):
        ''' Add a new node to the data store in the appropriate
            position in the tree. '''
        if parent is None:
            # If there isn't a root node, the new node becomes the root
            if self.rootElement is None:
                self.rootElement = node
            return self.rootElement
        if str(node) in parent.children:
            try:
                node.children = parent[str(node)].children
            except AttributeError:
                pass
            try:
                node.summaryData = parent[str(node)].summaryData
            except AttributeError:
                pass
        # Add the new node as a child of the parent
        parent[str(node)] = node
        return parent[str(node)]
        
    def getNode(self, ancestry=[]):
        ''' Find a node in the data store based on a node path '''
        # If no path was given, just return the root node.
        if not len(ancestry):
            return self.rootElement
        node = None
        # Follow the path given in the ancestry list until the 
        #  correct node is found.
        while ancestry:
            nodeId = ancestry.pop(0)
            if node is None:
                if nodeId == str(self.rootElement):
                    node = self.rootElement
                else: return None
            else:
                try:
                    node = node[nodeId]
                except KeyError:
                    pass
        return node

    def updateFinished(self, clusterNode):
        ''' This method is called when the gmond reader has finished updating
            a cluster.  It indicates that a summary can be done over the
            entire cluster and then the cluster transaction needs to be
            entered and passed to the plugins. '''
        if clusterNode is not None:
            if 'CLUSTER' == clusterNode.id:
                self._doClusterSummary(clusterNode);
            if 'GRID' == clusterNode.id:
                self._doGridSummary(clusterNode)
            self.notifier.insertTransaction(clusterNode)
        
    def acquireLock(self, obj):
        ''' Acquire a data store lock. '''
        self.lock.acquire()
        logging.debug('DataStore lock acquired %s'%str(obj))

    def releaseLock(self, obj):
        ''' Release the data store lock. ''' 
        self.lock.release()
        logging.debug('DataStore lock released%s'%str(obj))
Beispiel #3
0
class DataStore:
    '''The Datastore object stores all of the metric data
       as well as summary data for all data sources. It is
       a singleton object which guarantees that all instances
       of the object will produce the same data store. '''
    _shared_state = {} #Storage for a singleton object
    _initialized = False
    lock = thread.allocate_lock()
    
    def __init__(self):
        # Replace the objects attributes with the original
        #  attributes and data.
        self.__dict__ = DataStore._shared_state

        # Make sure that the DataStore object is only initialized once.
        if not DataStore._initialized:
            # Allocate a lock that will be used by all threads
            #  that are reading or writing to the data store.
            self.lock = thread.allocate_lock()
            self.lock.acquire()
            self.rootElement = None
            # Initialize the data store with the GANGLIA_XML and GRID tags.
            # Data store should never be completely empty.  Even if there are
            # no reporting data sources the web front end depends on having
            # at least a GANGLIA_XML tag and a nested GRID tag.
            cfg = getConfig()
            self.setNode(Element('GANGLIA_XML', {'VERSION':cfg.VERSION, 'SOURCE':'gmetad'}))
            self.setNode(Element('GRID', {'NAME':cfg[GmetadConfig.GRIDNAME], 'AUTHORITY':cfg[GmetadConfig.AUTHORITY], 'LOCALTIME':'%d' % time.time()}), self.rootElement)
            self.lock.release()

            # Start up the grid summary thread. 
            self.gridSummary = DataStoreGridSummary()
            self.gridSummary.start()

            # Start up the plugin notifier.
            self.notifier = GmetadNotifier()
            self.notifier.start()
            DataStore._initialized = True

    def _doGridSummary(self, gridNode):
        '''This private function will calculate the summaries for a
            single grid.  It is called each time new data
            has been read for a specific grid.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        gridNode.summaryData = {}
        gridNode.summaryData['summary'] = {}
        gridNode.summaryData['hosts_up'] = 0
        gridNode.summaryData['hosts_down'] = 0
        gridUp = (gridNode.getAttr('status') == 'up')
        summaryTime = int(time.time())

        # Summarize over each host contained by the cluster
        for clusterNode in gridNode:
            # Assume that cluster is up if we get a clusterNode object
            clusterUp = True
            for hostNode in clusterNode:
                reportedTime = summaryTime
                # Sum up the status of all of the hosts
                if 'HOST' == hostNode.id:
                    # Calculate the difference between the last known reported time
                    #  and the current time.  This determines if the host is up or down
                    # ** There may still be some issues with the way that this calculation is done
                    # ** The metric node below may also have the same issues.
                    reportedTime = int(hostNode.getAttr('reported'))
                    tn = int(hostNode.getAttr('tn'))
                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        hostNode.setAttr('tn', str(tn))
                    else:
                        hostNode.lastReportedTime = reportedTime

                    try:
                        if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4):
                            gridNode.summaryData['hosts_up'] += 1
                        else:
                            gridNode.summaryData['hosts_down'] += 1
                    except AttributeError:
                        pass
                    except KeyError:
                        pass
                # Summarize over each metric within a host
                for metricNode in hostNode:
                    tn = int(metricNode.getAttr('tn'))

                    # If the last reported time is the same as the current reported time, then
                    #  the host has not been updated.  Therefore calculate the time offset from
                    #  the current time.
                    if hostNode.lastReportedTime == reportedTime:
                        tn = summaryTime - reportedTime
                        if tn < 0: tn = 0
                        metricNode.setAttr('tn', str(tn))

                    # Don't include metrics that can not be summarized
                    if metricNode.getAttr('type') in ['string', 'timestamp']:
                        continue
                    try:
                        # Pull the existing summary node from the summary data
                        # dictionary. If one doesn't exist, add it in the exception.
                        summaryNode = gridNode.summaryData['summary'][str(metricNode)]
                        currSum = summaryNode.getAttr('sum')
                        summaryNode.incAttr('sum',  float(metricNode.getAttr('val')))
                    except KeyError:
                        # Since summary metrics use a different tag, create the new 
                        #  summary node with correct tag.
                        summaryNode = metricNode.summaryCopy(tag='METRICS')
                        # Initialize the first summary value and change the data type
                        # to double for all metric summaries
                        summaryNode.setAttr('sum', float(metricNode.getAttr('val')))
                        summaryNode.setAttr('type', 'double')
                        # Add the summary node to the summary dictionary
                        gridNode.summaryData['summary'][str(summaryNode)] = summaryNode
                    summaryNode.incAttr('num', 1)
        self.releaseLock(self)

    def _doClusterSummary(self, clusterNode):
        '''This private function will calculate the summaries for a
            single cluster.  It is called each time that new data
            has been read for a specific cluster.  All summary data
            is placed in the summaryData dictionary. '''
        self.acquireLock(self)

        # Clear out the summaryData from the last run and initialize
        #  the dictionary for new summaries.
        clusterNode.summaryData = {}
        clusterNode.summaryData['summary'] = {}
        clusterNode.summaryData['hosts_up'] = 0
        clusterNode.summaryData['hosts_down'] = 0
        clusterUp = (clusterNode.getAttr('status') == 'up')
        summaryTime = int(time.time())
        
        # Summarize over each host contained by the cluster
        for hostNode in clusterNode:
            reportedTime = summaryTime
            # Sum up the status of all of the hosts
            if 'HOST' == hostNode.id:
                # Calculate the difference between the last known reported time
                #  and the current time.  This determines if the host is up or down
                # ** There may still be some issues with the way that this calculation is done
                # ** The metric node below may also have the same issues.
                reportedTime = int(hostNode.getAttr('reported'))
                tn = int(hostNode.getAttr('tn'))
                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if hostNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    hostNode.setAttr('tn', str(tn))
                else:
                    hostNode.lastReportedTime = reportedTime
                    
                try:
                    if clusterUp and (int(hostNode.getAttr('tn')) < int(hostNode.getAttr('tmax'))*4):
                        clusterNode.summaryData['hosts_up'] += 1
                    else:
                        clusterNode.summaryData['hosts_down'] += 1
                except AttributeError:
                    pass
                except KeyError:
                    pass
            # Summarize over each metric within a host
            for metricNode in hostNode:
                tn = int(metricNode.getAttr('tn'))
                    
                # If the last reported time is the same as the current reported time, then
                #  the host has not been updated.  Therefore calculate the time offset from
                #  the current time.
                if metricNode.lastReportedTime == reportedTime:
                    tn = summaryTime - reportedTime
                    if tn < 0: tn = 0
                    metricNode.setAttr('tn', str(tn))
                    
                # Don't include metrics that can not be summarized
                if metricNode.getAttr('type') in ['string', 'timestamp']:
                    continue
                try:
                    # Pull the existing summary node from the summary data
                    # dictionary. If one doesn't exist, add it in the exception.
                    summaryNode = clusterNode.summaryData['summary'][str(metricNode)]
                    currSum = summaryNode.getAttr('sum')
                    try :
                        summaryNode.incAttr('sum',  float(metricNode.getAttr('val')))
                    except ValueError, msg:
                        print str(msg) + " sum: " + metricNode.getAttr('val')
                except KeyError:
                    # Since summary metrics use a different tag, create the new 
                    #  summary node with correct tag.
                    summaryNode = metricNode.summaryCopy(tag='METRICS')
                    # Initialize the first summary value and change the data type
                    # to double for all metric summaries
                    try :
                        summaryNode.setAttr('sum', float(metricNode.getAttr('val')))
                        summaryNode.setAttr('type', 'double')
                    except ValueError, msg:
                        print str(msg) + " sum: " + metricNode.getAttr('val')
                    # Add the summary node to the summary dictionary
                    clusterNode.summaryData['summary'][str(summaryNode)] = summaryNode
                summaryNode.incAttr('num', 1)
        self.releaseLock(self)