class SummarizeLogsAgent( AgentModule ):
""" SummarizeLogsAgent as extension of AgentModule.
"""
def __init__( self, *args, **kwargs ):
""" Constructor.
"""
AgentModule.__init__( self, *args, **kwargs )
self.rsClient = None
def initialize( self ):
""" Standard initialize.
:return: S_OK
"""
self.rsClient = ResourceStatusClient()
return S_OK()
def execute( self ):
""" execute ( main method )
The execute method runs over the three families of tables ( Site, Resource and
Node ) performing identical operations. First, selects all logs for a given
family ( and keeps track of which one is the last row ID ). It summarizes the
logs and finally, deletes the logs from the database.
:return: S_OK
"""
# loop over the tables
for element in ( 'Site', 'Resource', 'Node' ):
self.log.info( 'Summarizing %s' % element )
# get all logs to be summarized
selectLogElements = self._summarizeLogs( element )
if not selectLogElements[ 'OK' ]:
self.log.error( selectLogElements[ 'Message' ] )
continue
lastID, logElements = selectLogElements[ 'Value' ]
# logElements is a dictionary of key-value pairs as follows:
# ( name, statusType ) : list( logs )
for key, logs in logElements.iteritems():
sumResult = self._registerLogs( element, key, logs )
if not sumResult[ 'OK' ]:
self.log.error( sumResult[ 'Message' ] )
continue
if lastID is not None:
self.log.info( 'Deleting %sLog till ID %s' % ( element, lastID ) )
deleteResult = self.rsClient.deleteStatusElement( element, 'Log',
meta = { 'older' : ( 'ID', lastID ) } )
if not deleteResult[ 'OK' ]:
self.log.error( deleteResult[ 'Message' ] )
continue
return S_OK()
#.............................................................................
def _summarizeLogs( self, element ):
""" given an element, selects all logs in table <element>Log.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
:return: S_OK( lastID, listOfLogs ) / S_ERROR
"""
selectResults = self.rsClient.selectStatusElement( element, 'Log' )
if not selectResults[ 'OK' ]:
return selectResults
selectedItems = {}
selectColumns = selectResults[ 'Columns' ]
selectResults = selectResults[ 'Value' ]
latestID = None
if selectResults:
latestID = dict( zip( selectColumns, selectResults[ -1 ] ) )[ 'ID' ]
for selectResult in selectResults:
elementDict = dict( zip( selectColumns, selectResult ) )
key = ( elementDict[ 'Name' ], elementDict[ 'StatusType' ] )
if key not in selectedItems:
selectedItems[ key ] = [ elementDict ]
else:
lastStatus = selectedItems[ key ][ -1 ][ 'Status' ]
lastToken = selectedItems[ key ][ -1 ][ 'TokenOwner' ]
# If there are no changes on the Status or the TokenOwner with respect
# the previous one, discards the log.
if lastStatus != elementDict[ 'Status' ] or lastToken != elementDict[ 'TokenOwner' ]:
selectedItems[ key ].append( elementDict )
return S_OK( ( latestID, selectedItems ) )
def _registerLogs( self, element, key, logs ):
""" Given an element, a key - which is a tuple ( <name>, <statusType> )
and a list of dictionaries, this method inserts them on the <element>History
table. Before inserting them, checks whether the first one is or is not on
the <element>History table. If it is, it is not inserted. It also checks
whether the LastCheckTime parameter of the first log to be inserted is
larger than the last history log LastCheckTime. If not, it means an agent
cycle has been interrupted and we can run into inconsistencies. It aborts to
prevent more dramatic results.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**key** - `tuple`
tuple with the name of the element and the statusType
**logs** - `list`
list of dictionaries containing the logs
:return: S_OK / S_ERROR
"""
# Undo key
name, statusType = key
selectedRes = self.rsClient.selectStatusElement( element, 'History', name,
statusType,
meta = { 'columns' : [ 'Status', 'LastCheckTime', 'TokenOwner' ],
'limit' : 1,
'order' : ('LastCheckTime', 'DESC') } )
if not selectedRes[ 'OK' ]:
return selectedRes
selectedRes = selectedRes[ 'Value' ]
# We want from the <element>History table the last Status, LastCheckTime
# and TokenOwner
lastStatus, lastCheckTime, lastToken = None, None, None
if selectedRes:
lastStatus, lastCheckTime, lastToken = selectedRes[ 0 ]
# Sanity check to avoid running if an agent cycle has been stopped
if lastCheckTime and logs[ 0 ][ 'LastCheckTime' ] < lastCheckTime:
return S_ERROR( 'Overlapping data. Seems the DB has not been cleared properly' )
# If the first of the selected items has a different status than the latest
# on the history, we add it.
if logs[ 0 ][ 'Status' ] == lastStatus and logs[ 0 ][ 'TokenOwner' ] == lastToken:
logs.remove( logs[ 0 ] )
if logs:
self.log.info( '%s ( %s )' % ( name, statusType ) )
for selectedItemDict in logs:
res = self.__logToHistoryTable( element, selectedItemDict )
if not res[ 'OK' ]:
return res
return S_OK()
def __logToHistoryTable( self, element, elementDict ):
""" Given an element and a dictionary with all the arguments, this method
inserts a new entry on the <element>History table
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**elementDict** - `dict`
dictionary returned from the DB to be inserted on the History table
:return: S_OK / S_ERROR
"""
try:
name = elementDict[ 'Name' ]
statusType = elementDict[ 'StatusType' ]
status = elementDict[ 'Status' ]
elementType = elementDict[ 'ElementType' ]
reason = elementDict[ 'Reason' ]
dateEffective = elementDict[ 'DateEffective' ]
lastCheckTime = elementDict[ 'LastCheckTime' ]
tokenOwner = elementDict[ 'TokenOwner' ]
tokenExpiration = elementDict[ 'TokenExpiration' ]
except KeyError, e:
return S_ERROR( e )
self.log.info( ' %(Status)s %(DateEffective)s %(TokenOwner)s %(Reason)s' % elementDict )
return self.rsClient.insertStatusElement( element, 'History', name, statusType,
status, elementType, reason,
dateEffective, lastCheckTime,
tokenOwner, tokenExpiration )
class SummarizeLogsAgent(AgentModule):
""" SummarizeLogsAgent as extension of AgentModule.
"""
def __init__(self, *args, **kwargs):
""" Constructor.
"""
AgentModule.__init__(self, *args, **kwargs)
self.rsClient = None
def initialize(self):
""" Standard initialize.
:return: S_OK
"""
self.rsClient = ResourceStatusClient()
return S_OK()
def execute(self):
""" execute ( main method )
The execute method runs over the three families of tables ( Site, Resource and
Node ) performing identical operations. First, selects all logs for a given
family ( and keeps track of which one is the last row ID ). It summarizes the
logs and finally, deletes the logs from the database.
:return: S_OK
"""
# loop over the tables
for element in ('Site', 'Resource', 'Node'):
self.log.info('Summarizing %s' % element)
# get all logs to be summarized
selectLogElements = self._summarizeLogs(element)
if not selectLogElements['OK']:
self.log.error(selectLogElements['Message'])
continue
lastID, logElements = selectLogElements['Value']
# logElements is a dictionary of key-value pairs as follows:
# ( name, statusType ) : list( logs )
for key, logs in logElements.iteritems():
sumResult = self._registerLogs(element, key, logs)
if not sumResult['OK']:
self.log.error(sumResult['Message'])
continue
if lastID is not None:
self.log.info('Deleting %sLog till ID %s' % (element, lastID))
deleteResult = self.rsClient.deleteStatusElement(
element, 'Log', meta={'older': ['ID', lastID]})
if not deleteResult['OK']:
self.log.error(deleteResult['Message'])
continue
return S_OK()
def _summarizeLogs(self, element):
""" given an element, selects all logs in table <element>Log.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource or Node )
:return: S_OK( lastID, listOfLogs ) / S_ERROR
"""
selectResults = self.rsClient.selectStatusElement(element, 'Log')
if not selectResults['OK']:
return selectResults
selectedItems = {}
latestID = None
if not selectResults['Value']:
return S_OK((latestID, selectedItems))
selectColumns = selectResults['Columns']
selectResults = selectResults['Value']
if selectResults:
latestID = dict(zip(selectColumns, selectResults[-1]))['ID']
for selectResult in selectResults:
elementDict = dict(zip(selectColumns, selectResult))
key = (elementDict['Name'], elementDict['StatusType'])
if key not in selectedItems:
selectedItems[key] = [elementDict]
else:
lastStatus = selectedItems[key][-1]['Status']
lastToken = selectedItems[key][-1]['TokenOwner']
# If there are no changes on the Status or the TokenOwner with respect
# the previous one, discards the log.
if lastStatus != elementDict[
'Status'] or lastToken != elementDict['TokenOwner']:
selectedItems[key].append(elementDict)
return S_OK((latestID, selectedItems))
def _registerLogs(self, element, key, logs):
""" Given an element, a key - which is a tuple ( <name>, <statusType> )
and a list of dictionaries, this method inserts them on the <element>History
table. Before inserting them, checks whether the first one is or is not on
the <element>History table. If it is, it is not inserted.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**key** - `tuple`
tuple with the name of the element and the statusType
**logs** - `list`
list of dictionaries containing the logs
:return: S_OK / S_ERROR
"""
if not logs:
return S_OK()
# Undo key
name, statusType = key
selectedRes = self.rsClient.selectStatusElement(
element,
'History',
name,
statusType,
meta={
'columns': ['Status', 'TokenOwner'],
'limit': 1,
'order': ['DateEffective', 'desc']
})
if not selectedRes['OK']:
return selectedRes
selectedRes = selectedRes['Value']
if not selectedRes:
for selectedItemDict in logs:
res = self.__logToHistoryTable(element, selectedItemDict)
if not res['OK']:
return res
return S_OK()
# We want from the <element>History table the last Status, and TokenOwner
lastStatus, lastToken = None, None
if selectedRes:
try:
lastStatus = selectedRes[0][0]
lastToken = selectedRes[0][1]
except IndexError:
pass
# If the first of the selected items has a different status than the latest
# on the history, we keep it, otherwise we remove it.
if logs[0]['Status'] == lastStatus and logs[0][
'TokenOwner'] == lastToken:
logs.pop(0)
if logs:
self.log.info('%s ( %s ):' % (name, statusType))
self.log.debug(logs)
for selectedItemDict in logs:
res = self.__logToHistoryTable(element, selectedItemDict)
if not res['OK']:
return res
return S_OK()
def __logToHistoryTable(self, element, elementDict):
""" Given an element and a dictionary with all the arguments, this method
inserts a new entry on the <element>History table
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**elementDict** - `dict`
dictionary returned from the DB to be inserted on the History table
:return: S_OK / S_ERROR
"""
name = elementDict.get('Name')
statusType = elementDict.get('StatusType')
# vo = elementDict.get('VO') # FIXME: not sure about it
status = elementDict.get('Status')
elementType = elementDict.get('ElementType')
reason = elementDict.get('Reason')
dateEffective = elementDict.get('DateEffective')
lastCheckTime = elementDict.get('LastCheckTime')
tokenOwner = elementDict.get('TokenOwner')
tokenExpiration = elementDict.get('TokenExpiration')
self.log.info(' %s %s %s %s' %
(status, dateEffective, tokenOwner, reason))
return self.rsClient.insertStatusElement(
element=element,
tableType='History',
name=name,
statusType=statusType,
status=status,
elementType=elementType,
reason=reason,
dateEffective=dateEffective,
lastCheckTime=lastCheckTime,
tokenOwner=tokenOwner,
tokenExpiration=tokenExpiration)
class DatabaseCleanerAgent( AgentModule ):
'''
Agent that cleans the tables that may grow. With other words, it cleans from
old entries all the cache tables in ResourceManagementDB and the log and
history tables on the ResourceStatusDB.
'''
# Max number of minutes for the cache tables, 60 minutes by default
__maxCacheLifetime = 60
# Max number of days for the history tables
__maxHistoryLifetime = 10
# Max number of days for the log tables
__maxLogLifetime = 20
# List of caches to be processed
__cacheNames = ( 'DowntimeCache', 'GGUSTicketsCache', 'JobCache',
'PilotCache', 'TransferCache', 'SpaceTokenOccupancyCache' )
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.maxCacheLifetime = self.__maxCacheLifetime
self.maxHistoryLifetime = self.__maxHistoryLifetime
self.maxLogLifetime = self.__maxLogLifetime
self.rsClient = None
self.rmClient = None
def initialize( self ):
''' Standard initialize.
Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
'''
self.am_setOption( 'shifterProxy', 'ProductionManager' )
self.maxCacheLifetime = self.am_getOption( 'maxCacheLifetime', self.maxCacheLifetime )
self.maxHistoryLifetime = self.am_getOption( 'maxHistoryLifetime', self.maxHistoryLifetime )
self.maxLogLifetime = self.am_getOption( 'maxLogLifetime', self.maxLogLifetime )
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
return S_OK()
def execute( self ):
# TODO: uncomment when ResourceMonitoring is ready
# self._cleanCaches()
self._cleanStatusTable( 'History', self.maxHistoryLifetime )
self._cleanStatusTable( 'Log', self.maxLogLifetime )
return S_OK()
## Protected methods #########################################################
def _cleanCaches( self ):
'''
Method that iterates over all the caches in ResourceManagemetnDB deleting
entries with a LastCheckTime parameter older than now - X( hours ). On theory,
there should not be any parameter to be deleted. If there are, it means that
by some reason that entry has not been updated.
'''
self.log.info( 'Cleaning cache entries older than %s minutes' % self.maxCacheLifetime )
lastValidRecord = datetime.utcnow() - timedelta( minutes = self.maxCacheLifetime )
for cache in self.__cacheNames:
self.log.info( 'Inspecting %s' % cache )
deleteCache = 'delete%s' % cache
if not hasattr( self.rmClient, deleteCache ):
self.log.warn( '%s not found' % deleteCache )
continue
deleteMethod = getattr( self.rmClient, deleteCache )
deleteResults = deleteMethod( meta = { 'older' : ( 'LastCheckTime', lastValidRecord ) } )
if not deleteResults[ 'OK' ]:
self.log.error( deleteResults[ 'Message' ] )
continue
if deleteResults[ 'Value' ]:
self.log.info( 'Deleted %s entries' % deleteResults[ 'Value' ] )
else:
self.log.info( '... nothing to delete' )
return S_OK()
def _cleanStatusTable( self, tableType, lifeTime ):
'''
Method that deletes all entries older than now - lifeTime ( days ) for all
the elementType tables for a given tableType ( History / Log )
'''
self.log.info( 'Cleaning %s entries older than %s days' % ( tableType, lifeTime ) )
# It is hard-coded, mainly because there are no more tables going to be added
# to the schema for a long time.
elements = ( 'Site', 'Resource', 'Node' )
lastValidRecord = datetime.utcnow() - timedelta( days = lifeTime )
meta = { 'older' : ( 'LastCheckTime', lastValidRecord ) }
for element in elements:
self.log.info( 'Inspecting %s%s' % ( element, tableType ) )
deleteResults = self.rsClient.deleteStatusElement( element, tableType,
meta = meta )
if not deleteResults[ 'OK' ]:
self.log.error( deleteResults[ 'Message' ] )
continue
if deleteResults[ 'Value' ]:
self.log.info( 'Deleted %s entries' % deleteResults[ 'Value' ] )
else:
self.log.info( '... nothing to delete' )
return S_OK()
class DatabaseCleanerAgent(AgentModule):
'''
Agent that cleans the tables that may grow. With other words, it cleans from
old entries all the cache tables in ResourceManagementDB and the log and
history tables on the ResourceStatusDB.
'''
# Max number of minutes for the cache tables, 60 minutes by default
__maxCacheLifetime = 60
# Max number of days for the history tables
__maxHistoryLifetime = 10
# Max number of days for the log tables ( in principle, we do not delete ! )
__maxLogLifetime = 10000
# List of caches to be processed
__cacheNames = ('DowntimeCache', 'GGUSTicketsCache', 'JobCache',
'PilotCache', 'TransferCache', 'SpaceTokenOccupancyCache')
def __init__(self, *args, **kwargs):
''' c'tor
'''
AgentModule.__init__(self, *args, **kwargs)
self.maxCacheLifetime = self.__maxCacheLifetime
self.maxHistoryLifetime = self.__maxHistoryLifetime
self.maxLogLifetime = self.__maxLogLifetime
self.rsClient = None
self.rmClient = None
def initialize(self):
''' Standard initialize.
Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
'''
self.maxCacheLifetime = self.am_getOption('maxCacheLifetime',
self.maxCacheLifetime)
self.maxHistoryLifetime = self.am_getOption('maxHistoryLifetime',
self.maxHistoryLifetime)
self.maxLogLifetime = self.am_getOption('maxLogLifetime',
self.maxLogLifetime)
self.rsClient = ResourceStatusClient()
self.rmClient = ResourceManagementClient()
return S_OK()
def execute(self):
# TODO: uncomment when ResourceMonitoring is ready
# self._cleanCaches()
self._cleanStatusTable('History', self.maxHistoryLifetime)
self._cleanStatusTable('Log', self.maxLogLifetime)
return S_OK()
## Protected methods #########################################################
def _cleanCaches(self):
'''
Method that iterates over all the caches in ResourceManagemetnDB deleting
entries with a LastCheckTime parameter older than now - X( hours ). On theory,
there should not be any parameter to be deleted. If there are, it means that
by some reason that entry has not been updated.
'''
self.log.info('Cleaning cache entries older than %s minutes' %
self.maxCacheLifetime)
lastValidRecord = datetime.utcnow() - timedelta(
minutes=self.maxCacheLifetime)
for cache in self.__cacheNames:
self.log.info('Inspecting %s' % cache)
deleteCache = 'delete%s' % cache
if not hasattr(self.rmClient, deleteCache):
self.log.warn('%s not found' % deleteCache)
continue
deleteMethod = getattr(self.rmClient, deleteCache)
deleteResults = deleteMethod(
meta={'older': ('LastCheckTime', lastValidRecord)})
if not deleteResults['OK']:
self.log.error(deleteResults['Message'])
continue
if deleteResults['Value']:
self.log.info('Deleted %s entries' % deleteResults['Value'])
else:
self.log.info('... nothing to delete')
return S_OK()
def _cleanStatusTable(self, tableType, lifeTime):
'''
Method that deletes all entries older than now - lifeTime ( days ) for all
the elementType tables for a given tableType ( History / Log )
'''
self.log.info('Cleaning %s entries older than %s days' %
(tableType, lifeTime))
# It is hard-coded, mainly because there are no more tables going to be added
# to the schema for a long time.
elements = ('Site', 'Resource', 'Node')
lastValidRecord = datetime.utcnow() - timedelta(days=lifeTime)
meta = {'older': ('LastCheckTime', lastValidRecord)}
for element in elements:
self.log.info('Inspecting %s%s' % (element, tableType))
deleteResults = self.rsClient.deleteStatusElement(element,
tableType,
meta=meta)
if not deleteResults['OK']:
self.log.error(deleteResults['Message'])
continue
if deleteResults['Value']:
self.log.info('Deleted %s entries' % deleteResults['Value'])
else:
self.log.info('... nothing to delete')
return S_OK()
class SummarizeLogsAgent( AgentModule ):
""" SummarizeLogsAgent as extension of AgentModule.
"""
def __init__( self, *args, **kwargs ):
""" Constructor.
"""
AgentModule.__init__( self, *args, **kwargs )
self.rsClient = None
def initialize( self ):
""" Standard initialize.
:return: S_OK
"""
self.rsClient = ResourceStatusClient()
return S_OK()
def execute( self ):
""" execute ( main method )
The execute method runs over the three families of tables ( Site, Resource and
Node ) performing identical operations. First, selects all logs for a given
family ( and keeps track of which one is the last row ID ). It summarizes the
logs and finally, deletes the logs from the database.
:return: S_OK
"""
# loop over the tables
for element in ( 'Site', 'Resource', 'Node' ):
self.log.info( 'Summarizing %s' % element )
# get all logs to be summarized
selectLogElements = self._summarizeLogs( element )
if not selectLogElements[ 'OK' ]:
self.log.error( selectLogElements[ 'Message' ] )
continue
lastID, logElements = selectLogElements[ 'Value' ]
# logElements is a dictionary of key-value pairs as follows:
# ( name, statusType ) : list( logs )
for key, logs in logElements.iteritems():
sumResult = self._registerLogs( element, key, logs )
if not sumResult[ 'OK' ]:
self.log.error( sumResult[ 'Message' ] )
continue
if lastID is not None:
self.log.info( 'Deleting %sLog till ID %s' % ( element, lastID ) )
deleteResult = self.rsClient.deleteStatusElement( element, 'Log',
meta = { 'older' : ( 'ID', lastID ) } )
if not deleteResult[ 'OK' ]:
self.log.error( deleteResult[ 'Message' ] )
continue
return S_OK()
#.............................................................................
def _summarizeLogs( self, element ):
""" given an element, selects all logs in table <element>Log.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
:return: S_OK( lastID, listOfLogs ) / S_ERROR
"""
selectResults = self.rsClient.selectStatusElement( element, 'Log' )
if not selectResults[ 'OK' ]:
return selectResults
selectedItems = {}
selectColumns = selectResults[ 'Columns' ]
selectResults = selectResults[ 'Value' ]
latestID = None
if selectResults:
latestID = dict( zip( selectColumns, selectResults[ -1 ] ) )[ 'ID' ]
for selectResult in selectResults:
elementDict = dict( zip( selectColumns, selectResult ) )
key = ( elementDict[ 'Name' ], elementDict[ 'StatusType' ] )
if not key in selectedItems:
selectedItems[ key ] = [ elementDict ]
else:
lastStatus = selectedItems[ key ][ -1 ][ 'Status' ]
lastToken = selectedItems[ key ][ -1 ][ 'TokenOwner' ]
# If there are no changes on the Status or the TokenOwner with respect
# the previous one, discards the log.
if lastStatus != elementDict[ 'Status' ] or lastToken != elementDict[ 'TokenOwner' ]:
selectedItems[ key ].append( elementDict )
return S_OK( ( latestID, selectedItems ) )
def _registerLogs( self, element, key, logs ):
""" Given an element, a key - which is a tuple ( <name>, <statusType> )
and a list of dictionaries, this method inserts them on the <element>History
table. Before inserting them, checks whether the first one is or is not on
the <element>History table. If it is, it is not inserted. It also checks
whether the LastCheckTime parameter of the first log to be inserted is
larger than the last history log LastCheckTime. If not, it means an agent
cycle has been interrupted and we can run into inconsistencies. It aborts to
prevent more dramatic results.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**key** - `tuple`
tuple with the name of the element and the statusType
**logs** - `list`
list of dictionaries containing the logs
:return: S_OK / S_ERROR
"""
# Undo key
name, statusType = key
selectedRes = self.rsClient.selectStatusElement( element, 'History', name,
statusType,
meta = { 'columns' : [ 'Status', 'LastCheckTime', 'TokenOwner' ],
'limit' : 1,
'order' : 'LastCheckTime DESC' } )
if not selectedRes[ 'OK' ]:
return selectedRes
selectedRes = selectedRes[ 'Value' ]
# We want from the <element>History table the last Status, LastCheckTime
# and TokenOwner
lastStatus, lastCheckTime, lastToken = None, None, None
if selectedRes:
lastStatus, lastCheckTime, lastToken = selectedRes[ 0 ]
# Sanity check to avoid running if an agent cycle has been stopped
if lastCheckTime and logs[ 0 ][ 'LastCheckTime' ] < lastCheckTime:
return S_ERROR( 'Overlapping data. Seems the DB has not been cleared properly' )
# If the first of the selected items has a different status than the latest
# on the history, we add it.
if logs[ 0 ][ 'Status' ] == lastStatus and logs[ 0 ][ 'TokenOwner' ] == lastToken:
logs.remove( logs[ 0 ] )
if logs:
self.log.info( '%s ( %s )' % ( name, statusType ) )
for selectedItemDict in logs:
res = self.__logToHistoryTable( element, selectedItemDict )
if not res[ 'OK' ]:
return res
return S_OK()
def __logToHistoryTable( self, element, elementDict ):
""" Given an element and a dictionary with all the arguments, this method
inserts a new entry on the <element>History table
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**elementDict** - `dict`
dictionary returned from the DB to be inserted on the History table
:return: S_OK / S_ERROR
"""
try:
name = elementDict[ 'Name' ]
statusType = elementDict[ 'StatusType' ]
status = elementDict[ 'Status' ]
elementType = elementDict[ 'ElementType' ]
reason = elementDict[ 'Reason' ]
dateEffective = elementDict[ 'DateEffective' ]
lastCheckTime = elementDict[ 'LastCheckTime' ]
tokenOwner = elementDict[ 'TokenOwner' ]
tokenExpiration = elementDict[ 'TokenExpiration' ]
except KeyError, e:
return S_ERROR( e )
self.log.info( ' %(Status)s %(DateEffective)s %(TokenOwner)s %(Reason)s' % elementDict )
return self.rsClient.insertStatusElement( element, 'History', name, statusType,
status, elementType, reason,
dateEffective, lastCheckTime,
tokenOwner, tokenExpiration )
#...............................................................................
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class SummarizeLogsAgent(AgentModule):
"""SummarizeLogsAgent as extension of AgentModule."""
def __init__(self, *args, **kwargs):
"""Constructor."""
AgentModule.__init__(self, *args, **kwargs)
self.rsClient = None
self.months = 36
def initialize(self):
"""Standard initialize.
:return: S_OK
"""
self.rsClient = ResourceStatusClient()
self.months = self.am_getOption("Months", self.months)
return S_OK()
def execute(self):
"""execute (main method)
The execute method runs over the three families of tables (Site, Resource and
Node) performing identical operations. First, selects all logs for a given
family (and keeps track of which one is the last row ID). It summarizes the
logs and finally, deletes the logs from the database.
At last, this agent removes older entries from history tables
:return: S_OK
"""
# loop over the tables
for element in ("Site", "Resource", "Node"):
self.log.info("Summarizing %s" % element)
# get all logs to be summarized
selectLogElements = self._summarizeLogs(element)
if not selectLogElements["OK"]:
self.log.error(selectLogElements["Message"])
continue
lastID, logElements = selectLogElements["Value"]
# logElements is a dictionary of key-value pairs as follows:
# (name, statusType) : list(logs)
for key, logs in logElements.items():
sumResult = self._registerLogs(element, key, logs)
if not sumResult["OK"]:
self.log.error(sumResult["Message"])
continue
if lastID is not None:
self.log.info("Deleting %sLog till ID %s" % (element, lastID))
deleteResult = self.rsClient.deleteStatusElement(
element, "Log", meta={"older": ["ID", lastID]})
if not deleteResult["OK"]:
self.log.error(deleteResult["Message"])
continue
if self.months:
self._removeOldHistoryEntries(element, self.months)
return S_OK()
def _summarizeLogs(self, element):
"""given an element, selects all logs in table <element>Log.
:param str element: name of the table family (either Site, Resource or Node)
:return: S_OK(lastID, listOfLogs) / S_ERROR
"""
selectResults = self.rsClient.selectStatusElement(element, "Log")
if not selectResults["OK"]:
return selectResults
selectedItems = {}
latestID = None
if not selectResults["Value"]:
return S_OK((latestID, selectedItems))
selectColumns = selectResults["Columns"]
selectResults = selectResults["Value"]
if selectResults:
latestID = dict(zip(selectColumns, selectResults[-1]))["ID"]
for selectResult in selectResults:
elementDict = dict(zip(selectColumns, selectResult))
key = (elementDict["Name"], elementDict["StatusType"])
if key not in selectedItems:
selectedItems[key] = [elementDict]
else:
lastStatus = selectedItems[key][-1]["Status"]
lastToken = selectedItems[key][-1]["TokenOwner"]
# If there are no changes on the Status or the TokenOwner with respect
# the previous one, discards the log.
if lastStatus != elementDict[
"Status"] or lastToken != elementDict["TokenOwner"]:
selectedItems[key].append(elementDict)
return S_OK((latestID, selectedItems))
def _registerLogs(self, element, key, logs):
"""Given an element, a key - which is a tuple (<name>, <statusType>)
and a list of dictionaries, this method inserts them on the <element>History
table. Before inserting them, checks whether the first one is or is not on
the <element>History table. If it is, it is not inserted.
:param str element: name of the table family (either Site, Resource or Node)
:param tuple key: tuple with the name of the element and the statusType
:param list logs: list of dictionaries containing the logs
:return: S_OK(lastID, listOfLogs) / S_ERROR
:return: S_OK / S_ERROR
"""
if not logs:
return S_OK()
# Undo key
name, statusType = key
selectedRes = self.rsClient.selectStatusElement(
element,
"History",
name,
statusType,
meta={
"columns": ["Status", "TokenOwner"],
"limit": 1,
"order": ["DateEffective", "desc"]
},
)
if not selectedRes["OK"]:
return selectedRes
selectedRes = selectedRes["Value"]
if not selectedRes:
for selectedItemDict in logs:
res = self.__logToHistoryTable(element, selectedItemDict)
if not res["OK"]:
return res
return S_OK()
# We want from the <element>History table the last Status, and TokenOwner
lastStatus, lastToken = None, None
if selectedRes:
try:
lastStatus = selectedRes[0][0]
lastToken = selectedRes[0][1]
except IndexError:
pass
# If the first of the selected items has a different status than the latest
# on the history, we keep it, otherwise we remove it.
if logs[0]["Status"] == lastStatus and logs[0][
"TokenOwner"] == lastToken:
logs.pop(0)
if logs:
self.log.info("%s (%s):" % (name, statusType))
self.log.debug(logs)
for selectedItemDict in logs:
res = self.__logToHistoryTable(element, selectedItemDict)
if not res["OK"]:
return res
return S_OK()
def __logToHistoryTable(self, element, elementDict):
"""Given an element and a dictionary with all the arguments, this method
inserts a new entry on the <element>History table
:param str element: name of the table family (either Site, Resource or Node)
:param dict elementDict: dictionary returned from the DB to be inserted on the History table
:return: S_OK / S_ERROR
"""
name = elementDict.get("Name")
statusType = elementDict.get("StatusType")
# vo = elementDict.get('VO') # FIXME: not sure about it
status = elementDict.get("Status")
elementType = elementDict.get("ElementType")
reason = elementDict.get("Reason")
dateEffective = elementDict.get("DateEffective")
lastCheckTime = elementDict.get("LastCheckTime")
tokenOwner = elementDict.get("TokenOwner")
tokenExpiration = elementDict.get("TokenExpiration")
self.log.info(" %s %s %s %s" %
(status, dateEffective, tokenOwner, reason))
return self.rsClient.insertStatusElement(
element=element,
tableType="History",
name=name,
statusType=statusType,
status=status,
elementType=elementType,
reason=reason,
dateEffective=dateEffective,
lastCheckTime=lastCheckTime,
tokenOwner=tokenOwner,
tokenExpiration=tokenExpiration,
)
def _removeOldHistoryEntries(self, element, months):
"""Delete entries older than period
:param str element: name of the table family (either Site, Resource or Node)
:param int months: number of months
:return: S_OK / S_ERROR
"""
toRemove = datetime.utcnow().replace(microsecond=0) - timedelta(
days=30 * months)
self.log.info("Removing history entries", "older than %s" % toRemove)
deleteResult = self.rsClient.deleteStatusElement(
element, "History", meta={"older": ["DateEffective", toRemove]})
if not deleteResult["OK"]:
self.log.error(deleteResult["Message"])
class SummarizeLogsAgent(AgentModule):
""" SummarizeLogsAgent as extension of AgentModule.
"""
def __init__(self, *args, **kwargs):
""" Constructor.
"""
AgentModule.__init__(self, *args, **kwargs)
self.rsClient = None
def initialize(self):
""" Standard initialize.
:return: S_OK
"""
self.rsClient = ResourceStatusClient()
return S_OK()
def execute(self):
""" execute ( main method )
The execute method runs over the three families of tables ( Site, Resource and
Node ) performing identical operations. First, selects all logs for a given
family ( and keeps track of which one is the last row ID ). It summarizes the
logs and finally, deletes the logs from the database.
:return: S_OK
"""
# loop over the tables
for element in ('Site', 'Resource', 'Node'):
self.log.info('Summarizing %s' % element)
# get all logs to be summarized
selectLogElements = self._summarizeLogs(element)
if not selectLogElements['OK']:
self.log.error(selectLogElements['Message'])
continue
lastID, logElements = selectLogElements['Value']
# logElements is a dictionary of key-value pairs as follows:
# ( name, statusType ) : list( logs )
for key, logs in logElements.iteritems():
sumResult = self._registerLogs(element, key, logs)
if not sumResult['OK']:
self.log.error(sumResult['Message'])
continue
if lastID is not None:
self.log.info('Deleting %sLog till ID %s' % (element, lastID))
deleteResult = self.rsClient.deleteStatusElement(element, 'Log',
meta={'older': ('ID', lastID)})
if not deleteResult['OK']:
self.log.error(deleteResult['Message'])
continue
return S_OK()
def _summarizeLogs(self, element):
""" given an element, selects all logs in table <element>Log.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource or Node )
:return: S_OK( lastID, listOfLogs ) / S_ERROR
"""
selectResults = self.rsClient.selectStatusElement(element, 'Log')
if not selectResults['OK']:
return selectResults
selectedItems = {}
latestID = None
if not selectResults['Value']:
return S_OK((latestID, selectedItems))
selectColumns = selectResults['Columns']
selectResults = selectResults['Value']
if selectResults:
latestID = dict(zip(selectColumns, selectResults[-1]))['ID']
for selectResult in selectResults:
elementDict = dict(zip(selectColumns, selectResult))
key = (elementDict['Name'], elementDict['StatusType'])
if key not in selectedItems:
selectedItems[key] = [elementDict]
else:
lastStatus = selectedItems[key][-1]['Status']
lastToken = selectedItems[key][-1]['TokenOwner']
# If there are no changes on the Status or the TokenOwner with respect
# the previous one, discards the log.
if lastStatus != elementDict['Status'] or lastToken != elementDict['TokenOwner']:
selectedItems[key].append(elementDict)
return S_OK((latestID, selectedItems))
def _registerLogs(self, element, key, logs):
""" Given an element, a key - which is a tuple ( <name>, <statusType> )
and a list of dictionaries, this method inserts them on the <element>History
table. Before inserting them, checks whether the first one is or is not on
the <element>History table. If it is, it is not inserted.
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**key** - `tuple`
tuple with the name of the element and the statusType
**logs** - `list`
list of dictionaries containing the logs
:return: S_OK / S_ERROR
"""
if not logs:
return S_OK()
# Undo key
name, statusType = key
selectedRes = self.rsClient.selectStatusElement(element, 'History', name,
statusType,
meta={'columns': ['Status', 'TokenOwner'],
'limit': 1,
'order': ('DateEffective', 'desc')})
if not selectedRes['OK']:
return selectedRes
selectedRes = selectedRes['Value']
if not selectedRes:
return S_OK()
# We want from the <element>History table the last Status, and TokenOwner
lastStatus, lastToken = None, None
if selectedRes:
try:
lastStatus = selectedRes[0][0]
lastToken = selectedRes[0][1]
except IndexError:
pass
# If the first of the selected items has a different status than the latest
# on the history, we keep it, otherwise we remove it.
if logs[0]['Status'] == lastStatus and logs[0]['TokenOwner'] == lastToken:
logs.pop(0)
if logs:
self.log.info('%s ( %s ):' % (name, statusType))
self.log.debug(logs)
for selectedItemDict in logs:
res = self.__logToHistoryTable(element, selectedItemDict)
if not res['OK']:
return res
return S_OK()
def __logToHistoryTable(self, element, elementDict):
""" Given an element and a dictionary with all the arguments, this method
inserts a new entry on the <element>History table
:Parameters:
**element** - `string`
name of the table family ( either Site, Resource and Node )
**elementDict** - `dict`
dictionary returned from the DB to be inserted on the History table
:return: S_OK / S_ERROR
"""
name = elementDict.get('Name')
statusType = elementDict.get('StatusType')
status = elementDict.get('Status')
elementType = elementDict.get('ElementType')
reason = elementDict.get('Reason')
dateEffective = elementDict.get('DateEffective')
lastCheckTime = elementDict.get('LastCheckTime')
tokenOwner = elementDict.get('TokenOwner')
tokenExpiration = elementDict.get('TokenExpiration')
self.log.info(' %s %s %s %s' % (status, dateEffective, tokenOwner, reason))
return self.rsClient.insertStatusElement(element, 'History', name, statusType,
status, elementType, reason,
dateEffective, lastCheckTime,
tokenOwner, tokenExpiration)
