def export_checkComponentLog(self, component):
""" Check component log for errors
"""
componentList = []
if '*' in component:
if component == '*':
result = gComponentInstaller.getSetupComponents()
if result['OK']:
for ctype in ['Services', 'Agents', 'Executors']:
if ctype in result['Value']:
for sname in result['Value'][ctype]:
for cname in result['Value'][ctype][sname]:
componentList.append('/'.join(
[sname, cname]))
elif isinstance(component, basestring):
componentList = [component]
else:
componentList = component
resultDict = {}
for c in componentList:
if not '/' in c:
continue
system, cname = c.split('/')
startDir = gComponentInstaller.startDir
currentLog = startDir + '/' + system + '_' + cname + '/log/current'
logFile = file(currentLog, 'r')
logLines = logFile.readlines()
logFile.close()
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ''
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
timeStamp = fromString(fields[0] + ' ' + fields[1])
if (now - timeStamp) < hour:
errors_1 += 1
recent = True
if (now - timeStamp) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split('ERROR:')[-1].strip()
resultDict[c] = {
'ErrorsHour': errors_1,
'ErrorsDay': errors_24,
'LastError': lastError
}
return S_OK(resultDict)
def export_checkComponentLog( self, component ):
""" Check component log for errors
"""
componentList = []
if '*' in component:
if component == '*':
result = gComponentInstaller.getSetupComponents()
if result['OK']:
for ctype in ['Services', 'Agents', 'Executors']:
if ctype in result['Value']:
for sname in result['Value'][ctype]:
for cname in result['Value'][ctype][sname]:
componentList.append( '/'.join( [sname, cname] ) )
elif isinstance( component, basestring):
componentList = [component]
else:
componentList = component
resultDict = {}
for c in componentList:
if not '/' in c:
continue
system, cname = c.split( '/' )
startDir = gComponentInstaller.startDir
currentLog = startDir + '/' + system + '_' + cname + '/log/current'
logFile = file( currentLog, 'r' )
logLines = logFile.readlines()
logFile.close()
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ''
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
timeStamp = fromString( fields[0] + ' ' + fields[1] )
if ( now - timeStamp ) < hour:
errors_1 += 1
recent = True
if ( now - timeStamp ) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split( 'ERROR:' )[-1].strip()
resultDict[c] = {'ErrorsHour':errors_1, 'ErrorsDay':errors_24, 'LastError':lastError}
return S_OK( resultDict )
def export_checkComponentLog(self, component):
""" Check component log for errors
"""
componentList = []
if "*" in component:
if component == "*":
result = gComponentInstaller.getSetupComponents()
if result["OK"]:
for ctype in ["Services", "Agents", "Executors"]:
if ctype in result["Value"]:
for sname in result["Value"][ctype]:
for cname in result["Value"][ctype][sname]:
componentList.append("/".join([sname, cname]))
elif type(component) in StringTypes:
componentList = [component]
else:
componentList = component
resultDict = {}
for c in componentList:
if not "/" in c:
continue
system, cname = c.split("/")
startDir = gComponentInstaller.startDir
currentLog = startDir + "/" + system + "_" + cname + "/log/current"
logFile = file(currentLog, "r")
logLines = logFile.readlines()
logFile.close()
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ""
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
timeStamp = fromString(fields[0] + " " + fields[1])
if (now - timeStamp) < hour:
errors_1 += 1
recent = True
if (now - timeStamp) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split("ERROR:")[-1].strip()
resultDict[c] = {"ErrorsHour": errors_1, "ErrorsDay": errors_24, "LastError": lastError}
return S_OK(resultDict)
def __storeProfiling():
"""
Retrieves and stores into ElasticSearch profiling information about the components on the host
"""
# TODO: if we have a component which is not running, we will not profile the running processes
result = gComponentInstaller.getStartupComponentStatus([])
if not result['OK']:
gLogger.error(result['Message'])
return S_ERROR(result['Message'])
startupComps = result['Value']
result = gComponentInstaller.getSetupComponents()
if not result['OK']:
gLogger.error(result['Message'])
return S_ERROR(result['Message'])
setupComps = result['Value']
# Get the profiling information for every running component and send it to MonitoringSystem
for cType in setupComps:
for system in setupComps[cType]:
for comp in setupComps[cType][system]:
instance = "%s_%s" % (system, comp)
if instance not in startupComps:
gLogger.error("Wrongly configured component: %s" %
instance)
continue
pid = startupComps[instance]['PID']
if pid not in gProfilers:
gProfilers[pid] = Profiler.Profiler(pid)
profiler = gProfilers[pid]
result = profiler.getAllProcessData()
if result['OK']:
log = result['Value']['stats']
log['host'] = socket.getfqdn()
log['component'] = instance
log['timestamp'] = result['Value']['datetime']
gMonitoringReporter.addRecord(log)
else:
gLogger.error(result['Message'])
return result
gMonitoringReporter.commit()
return S_OK('Profiling information logged correctly')
def __storeProfiling():
"""
Retrieves and stores into ElasticSearch profiling information about the components on the host
"""
# TODO: if we have a component which is not running, we will not profile the running processes
result = gComponentInstaller.getStartupComponentStatus([])
if not result['OK']:
gLogger.error(result['Message'])
return S_ERROR(result['Message'])
startupComps = result['Value']
result = gComponentInstaller.getSetupComponents()
if not result['OK']:
gLogger.error(result['Message'])
return S_ERROR(result['Message'])
setupComps = result['Value']
# Get the profiling information for every running component and send it to MonitoringSystem
for cType in setupComps:
for system in setupComps[cType]:
for comp in setupComps[cType][system]:
instance = "%s_%s" % (system, comp)
if instance not in startupComps:
gLogger.error("Wrongly configured component: %s" % instance)
continue
pid = startupComps[instance]['PID']
if pid not in gProfilers:
gProfilers[pid] = Profiler.Profiler(pid)
profiler = gProfilers[pid]
result = profiler.getAllProcessData()
if result['OK']:
log = result['Value']['stats']
log['host'] = socket.getfqdn()
log['component'] = instance
log['timestamp'] = result['Value']['datetime']
gMonitoringReporter.addRecord(log)
else:
gLogger.error(result['Message'])
return result
gMonitoringReporter.commit()
return S_OK('Profiling information logged correctly')
def export_getUsedPorts( self ):
"""
Retrieve the ports in use by services on this host
:return: Returns a dictionary containing, for each system, which port is being used by which service
"""
result = gComponentInstaller.getSetupComponents()
if not result[ 'OK' ]:
return result
services = result[ 'Value' ][ 'Services' ]
ports = {}
for system in services:
ports[ system ] = {}
for service in services[ system ]:
url = PathFinder.getServiceURL( '%s/%s' % ( system, service ) )
port = re.search( r':(\d{4,5})/', url )
if port:
ports[ system ][ service ] = port.group( 1 )
else:
ports[ system ][ service ] = 'None'
return S_OK( ports )
def export_checkComponentLog(self, component):
""" Check component log for errors
"""
componentList = []
if '*' in component:
if component == '*':
result = gComponentInstaller.getSetupComponents()
if result['OK']:
for ctype in ['Services', 'Agents', 'Executors']:
if ctype in result['Value']:
for sname in result['Value'][ctype]:
for cname in result['Value'][ctype][sname]:
componentList.append('/'.join(
[sname, cname]))
elif isinstance(component, basestring):
componentList = [component]
else:
componentList = component
resultDict = {}
for comp in componentList:
if '/' not in comp:
continue
system, cname = comp.split('/')
startDir = gComponentInstaller.startDir
currentLog = startDir + '/' + system + '_' + cname + '/log/current'
try:
logFile = file(currentLog, 'r')
except IOError as err:
gLogger.error("File does not exists:", currentLog)
resultDict[comp] = {
'ErrorsHour': -1,
'ErrorsDay': -1,
'LastError': currentLog + '::' + repr(err)
}
continue
logLines = logFile.readlines()
logFile.close()
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ''
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
if len(fields) < 2: # if the line contains only one word
lastError = line.split('ERROR:')[-1].strip()
continue
timeStamp = fromString(fields[0] + ' ' + fields[1])
if not timeStamp: # if the timestamp is missing in the log
lastError = line.split('ERROR:')[-1].strip()
continue
if (now - timeStamp) < hour:
errors_1 += 1
recent = True
if (now - timeStamp) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split('ERROR:')[-1].strip()
resultDict[comp] = {
'ErrorsHour': errors_1,
'ErrorsDay': errors_24,
'LastError': lastError
}
return S_OK(resultDict)
def export_getSetupComponents(self):
""" Get the list of all the components ( services and agents )
set up for running with runsvdir in /opt/dirac/startup directory
"""
return gComponentInstaller.getSetupComponents()
def export_getSetupComponents( self ):
""" Get the list of all the components ( services and agents )
set up for running with runsvdir in /opt/dirac/startup directory
"""
return gComponentInstaller.getSetupComponents()
def export_checkComponentLog(self, component):
"""Check component log for errors"""
componentList = []
if "*" in component:
if component == "*":
result = gComponentInstaller.getSetupComponents()
if result["OK"]:
for ctype in ["Services", "Agents", "Executors"]:
if ctype in result["Value"]:
for sname in result["Value"][ctype]:
for cname in result["Value"][ctype][sname]:
componentList.append("/".join(
[sname, cname]))
elif isinstance(component, six.string_types):
componentList = [component]
else:
componentList = component
resultDict = {}
for comp in componentList:
if "/" not in comp:
continue
system, cname = comp.split("/")
startDir = gComponentInstaller.startDir
currentLog = startDir + "/" + system + "_" + cname + "/log/current"
try:
with open(currentLog, "r") as logFile:
logLines = logFile.readlines()
except IOError as err:
gLogger.error("File does not exists:", currentLog)
resultDict[comp] = {
"ErrorsHour": -1,
"ErrorsDay": -1,
"LastError": currentLog + "::" + repr(err)
}
continue
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ""
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
if len(fields) < 2: # if the line contains only one word
lastError = line.split("ERROR:")[-1].strip()
continue
timeStamp = fromString(fields[0] + " " + fields[1])
if not timeStamp: # if the timestamp is missing in the log
lastError = line.split("ERROR:")[-1].strip()
continue
if (now - timeStamp) < hour:
errors_1 += 1
recent = True
if (now - timeStamp) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split("ERROR:")[-1].strip()
resultDict[comp] = {
"ErrorsHour": errors_1,
"ErrorsDay": errors_24,
"LastError": lastError
}
return S_OK(resultDict)
def export_checkComponentLog(self, component):
""" Check component log for errors
"""
componentList = []
if '*' in component:
if component == '*':
result = gComponentInstaller.getSetupComponents()
if result['OK']:
for ctype in ['Services', 'Agents', 'Executors']:
if ctype in result['Value']:
for sname in result['Value'][ctype]:
for cname in result['Value'][ctype][sname]:
componentList.append('/'.join([sname, cname]))
elif isinstance(component, basestring):
componentList = [component]
else:
componentList = component
resultDict = {}
for comp in componentList:
if '/' not in comp:
continue
system, cname = comp.split('/')
startDir = gComponentInstaller.startDir
currentLog = startDir + '/' + system + '_' + cname + '/log/current'
try:
logFile = file(currentLog, 'r')
except IOError as err:
gLogger.error("File does not exists:", currentLog)
resultDict[comp] = {'ErrorsHour': -1, 'ErrorsDay': -1, 'LastError': currentLog + '::' + repr(err)}
continue
logLines = logFile.readlines()
logFile.close()
errors_1 = 0
errors_24 = 0
now = dateTime()
lastError = ''
for line in logLines:
if "ERROR:" in line:
fields = line.split()
recent = False
if len(fields) < 2: # if the line contains only one word
lastError = line.split('ERROR:')[-1].strip()
continue
timeStamp = fromString(fields[0] + ' ' + fields[1])
if not timeStamp: # if the timestamp is missing in the log
lastError = line.split('ERROR:')[-1].strip()
continue
if (now - timeStamp) < hour:
errors_1 += 1
recent = True
if (now - timeStamp) < day:
errors_24 += 1
recent = True
if recent:
lastError = line.split('ERROR:')[-1].strip()
resultDict[comp] = {'ErrorsHour': errors_1, 'ErrorsDay': errors_24, 'LastError': lastError}
return S_OK(resultDict)