def buildDbClient(self, dbClient):
if not dbClient:
raise ValueError('DB Client is not specified')
host = dbClient.getHost()
hostOsh = modeling.createHostOSH(host)
processOsh = modeling.createProcessOSH(dbClient.getName(), hostOsh)
return processOsh
def addProcess(pdu, hostId, cleanCommand, pid, commandLine, commAndPath, cleanArgs,
processList, discoverProcesses, myVec, hostOSH, startuptime = None, owner = None):
pdu.addProcess(hostId, cleanCommand, pid, commandLine, commAndPath, cleanArgs, owner, startuptime)
processID = cleanCommand
if commandLine != None:
processID = processID + '->' + commandLine
if ((processID in processList) != 0):
logger.debug('process: ',cleanCommand,' already reported..')
return
processList.append(processID)
if discoverProcesses:
processOsh = modeling.createProcessOSH(cleanCommand, hostOSH, commandLine, pid, commAndPath, cleanArgs, owner, startuptime)
myVec.add(processOsh)
def populateResult(self, oshv, hostName, userName, dbOSH, clientsCount):
if (hostName) and (userName):
#create the remote host
remoteHost = Util.getHost(hostName)
if not remoteHost:
logger.debug('RemoteHost osh is None, hostName:%s' % hostName)
return
oshv.add(remoteHost)
#create the remote process
program = modeling.createProcessOSH(userName, remoteHost)
oshv.add(program)
if dbOSH:
#create the dblink
dbLink = modeling.createLinkOSH('dbclient', dbOSH, program)
dbLink.setIntegerAttribute('dbclient_connectioncount',clientsCount)
oshv.add(dbLink)
else:
logger.debug('Database osh is None')
def discoverProcessesByWmic(client, OSHVResult, hostID, Framework, pid2Process = None):
''' Discover system processes, report them and save in probe DB.
Shell, oshVector, str, Framework, map[str, str] -> bool
@command: wmic process get commandLine, creationdate, executablepath, name, processId
'''
wmiProvider = wmiutils.getWmiProvider(client)
queryBuilder = wmiProvider.getBuilder('Win32_Process')
queryBuilder.usePathCommand(1)
#queryBuilder = wmiutils.WmicQueryBuilder('process')
queryBuilder.addWmiObjectProperties('name', 'processId', 'commandLine', 'executablepath', 'creationdate')
wmicAgent = wmiProvider.getAgent()
processItems = []
try:
processItems = wmicAgent.getWmiData(queryBuilder)
except:
logger.debugException('Failed getting processes information via wmic' )
return 0
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
processList = []
hostOSH = None
count = 0
for processItem in processItems:
if not processItem.name:
continue
processName = processItem.name
processNameLower = processName.lower()
processPid = processItem.processId
if processPid == '-1' or not processPid.isnumeric():
logger.debug("Process '%s' is system process or has non numeric pid" % processName)
continue
processExecutablePath = processItem.executablepath
processCommandLine = processItem.commandLine
processStartupTimeString = processItem.creationdate
processStartupTime = None
if processStartupTimeString:
try:
startupDate = modeling.getDateFromUtcString(processStartupTimeString)
processStartupTime = startupDate.getTime()
except:
errobj = errorobject.createError(errorcodes.PROCESS_STARTUP_TIME_ATTR_NOT_SET, ['NTCMD', processStartupTimeString], "%s: Process startup time attribute is not set due to error while parsing date string '%s'" % ('NTCMD', processStartupTimeString))
logger.reportWarningObject(errobj)
# check whether process name is included in command line
# Obtain first token containing process from the CMD line
matchObj = re.match('(:?["\'](.*?)["\']|(.*?)\s)', processCommandLine)
if matchObj and matchObj.groups():
firstCmdToken = matchObj.group(1).strip()
else:
firstCmdToken = processCommandLine.strip()
#remove quotes
firstCmdToken = re.sub('[\'"]', '', firstCmdToken).lower()
#token has to end with process name
if not firstCmdToken.endswith(processNameLower):
extStartPos = processNameLower.rfind('.')
if extStartPos != -1:
pnameNoExt = processNameLower[0:extStartPos]
if not firstCmdToken.endswith(pnameNoExt):
processCommandLine = '%s %s' % (processName, processCommandLine)
processArgs = None
argsMatch = re.match('("[^"]+"|[^"]\S+)\s+(.+)$',processCommandLine)
if argsMatch:
processArgs = argsMatch.group(2)
pdu.addProcess(hostID, processName, processPid, processCommandLine, processExecutablePath, processArgs, None, processStartupTime)
if processPid in processList:
logger.debug("Process: '%s' already reported" % processName)
continue
count += 1
processList.append(processPid)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString('host', hostID)
processOsh = modeling.createProcessOSH(processName, hostOSH, processCommandLine, processPid, processExecutablePath, None, None, processStartupTime)
OSHVResult.add(processOsh)
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
return 1
def discoverProcesses(client, OSHVResult, hostID, Framework, pid2Process = None):
cmdProcessInfo = 'processlist'
ntcmdErrStr = 'Remote command returned 1(0x1)'
localFile = CollectorsParameters.BASE_PROBE_MGR_DIR + CollectorsParameters.getDiscoveryResourceFolder() + CollectorsParameters.FILE_SEPARATOR + 'processlist.exe'
remoteFile = client.copyFileIfNeeded(localFile)
if not remoteFile:
logger.warn('Failed copying %s' % cmdProcessInfo)
return 0
buffer = client.execCmd(remoteFile)#V@@CMD_PERMISION ntcmd protocol execution
logger.debug('Output of ', remoteFile, ': ', buffer)
if buffer.find(ntcmdErrStr) != -1:
logger.warn('Failed getting process info')
else:
logger.debug('Got process info - parsing...')
processes = buffer.split('\n')
processList = []
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
hostOSH = None
count = 0
for process in processes:
process = process.strip()
name = ''
nameLower = ''
pid = '-1'
try:
# Get process name
matchName = re.search('\d*\s(.+)', process)
if matchName:
name = matchName.group(1)
name = name.strip()
nameLower = name.lower()
if name == '[System Process]':
continue
# Get free space
matchPid = re.search('(\d+)\s.*', process)
if matchPid:
pid = matchPid.group(1)
if(pid != '-1' and pid.isnumeric()):
pdu.addProcess(hostID, name, pid)
if ((pid in processList) != 0):
logger.debug('process: ',name,' already reported..')
continue
count = count + 1
processList.append(pid)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString('host', hostID)
processOsh = modeling.createProcessOSH(name, hostOSH, None, pid)
OSHVResult.add(processOsh)
else:
logger.debug('process: ',name,' is system process or has non numeric pid')
except:
logger.errorException('Error in discoverProcesses()')
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
logger.debug("Discovered ", str(count), " processes")
return 1
def _buildProcess(self, node):
return modeling.createProcessOSH(node.process.name,
node.hostOsh, node.process.cmdline,
node.process.pid, node.process.path,
node.process.params, node.process.owner,
node.process.startuptime)
def discoverProcessesByWmic(client,
OSHVResult,
hostID,
Framework,
pid2Process=None):
''' Discover system processes, report them and save in probe DB.
Shell, oshVector, str, Framework, map[str, str] -> bool
@command: wmic process get commandLine, creationdate, executablepath, name, processId
'''
wmiProvider = wmiutils.getWmiProvider(client)
queryBuilder = wmiProvider.getBuilder('Win32_Process')
queryBuilder.usePathCommand(1)
#queryBuilder = wmiutils.WmicQueryBuilder('process')
queryBuilder.addWmiObjectProperties('name', 'processId', 'commandLine',
'executablepath', 'creationdate')
wmicAgent = wmiProvider.getAgent()
processItems = []
try:
processItems = wmicAgent.getWmiData(queryBuilder)
except:
logger.debugException('Failed getting processes information via wmic')
return 0
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
processList = []
hostOSH = None
count = 0
for processItem in processItems:
if not processItem.name:
continue
processName = processItem.name
processNameLower = processName.lower()
processPid = processItem.processId
if processPid == '-1' or not processPid.isnumeric():
logger.debug(
"Process '%s' is system process or has non numeric pid" %
processName)
continue
processExecutablePath = processItem.executablepath
processCommandLine = processItem.commandLine
processStartupTimeString = processItem.creationdate
processStartupTime = None
if processStartupTimeString:
try:
startupDate = modeling.getDateFromUtcString(
processStartupTimeString)
processStartupTime = startupDate.getTime()
except:
errobj = errorobject.createError(
errorcodes.PROCESS_STARTUP_TIME_ATTR_NOT_SET,
['NTCMD', processStartupTimeString],
"%s: Process startup time attribute is not set due to error while parsing date string '%s'"
% ('NTCMD', processStartupTimeString))
logger.reportWarningObject(errobj)
# check whether process name is included in command line
# Obtain first token containing process from the CMD line
matchObj = re.match('(:?["\'](.*?)["\']|(.*?)\s)',
processCommandLine)
if matchObj and matchObj.groups():
firstCmdToken = matchObj.group(1).strip()
else:
firstCmdToken = processCommandLine.strip()
#remove quotes
firstCmdToken = re.sub('[\'"]', '', firstCmdToken).lower()
#token has to end with process name
if not firstCmdToken.endswith(processNameLower):
extStartPos = processNameLower.rfind('.')
if extStartPos != -1:
pnameNoExt = processNameLower[0:extStartPos]
if not firstCmdToken.endswith(pnameNoExt):
processCommandLine = '%s %s' % (processName,
processCommandLine)
processArgs = None
argsMatch = re.match('("[^"]+"|[^"]\S+)\s+(.+)$',
processCommandLine)
if argsMatch:
processArgs = argsMatch.group(2)
pdu.addProcess(hostID, processName, processPid, processCommandLine,
processExecutablePath, processArgs, None,
processStartupTime)
if processPid in processList:
logger.debug("Process: '%s' already reported" % processName)
continue
count += 1
processList.append(processPid)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString('host', hostID)
processOsh = modeling.createProcessOSH(
processName, hostOSH, processCommandLine, processPid,
processExecutablePath, None, None, processStartupTime)
OSHVResult.add(processOsh)
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
return 1
def discoverProcesses(client, OSHVResult, hostID, Framework, pid2Process=None):
cmdProcessInfo = 'processlist'
ntcmdErrStr = 'Remote command returned 1(0x1)'
localFile = CollectorsParameters.BASE_PROBE_MGR_DIR + CollectorsParameters.getDiscoveryResourceFolder(
) + CollectorsParameters.FILE_SEPARATOR + 'processlist.exe'
remoteFile = client.copyFileIfNeeded(localFile)
if not remoteFile:
logger.warn('Failed copying %s' % cmdProcessInfo)
return 0
buffer = client.execCmd(
remoteFile) #V@@CMD_PERMISION ntcmd protocol execution
logger.debug('Output of ', remoteFile, ': ', buffer)
if buffer.find(ntcmdErrStr) != -1:
logger.warn('Failed getting process info')
else:
logger.debug('Got process info - parsing...')
processes = buffer.split('\n')
processList = []
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
hostOSH = None
count = 0
for process in processes:
process = process.strip()
name = ''
nameLower = ''
pid = '-1'
try:
# Get process name
matchName = re.search('\d*\s(.+)', process)
if matchName:
name = matchName.group(1)
name = name.strip()
nameLower = name.lower()
if name == '[System Process]':
continue
# Get free space
matchPid = re.search('(\d+)\s.*', process)
if matchPid:
pid = matchPid.group(1)
if (pid != '-1' and pid.isnumeric()):
pdu.addProcess(hostID, name, pid)
if ((pid in processList) != 0):
logger.debug('process: ', name,
' already reported..')
continue
count = count + 1
processList.append(pid)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString(
'host', hostID)
processOsh = modeling.createProcessOSH(
name, hostOSH, None, pid)
OSHVResult.add(processOsh)
else:
logger.debug(
'process: ', name,
' is system process or has non numeric pid')
except:
logger.errorException('Error in discoverProcesses()')
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
logger.debug("Discovered ", str(count), " processes")
return 1
def doQueryMainframeTCP(client, OSHVResult, Framework):
ip_address = client.getIpAddress()
hostOSH = modeling.createHostOSH(client.getIpAddress())
modeling.setHostOsFamily(hostOSH, 'mainframe')
################ Query and data ################################
resultSetList = client.executeQuery(
'1.3.6.1.4.1.2.6.19.2.2.7.1.1.37,1.3.6.1.4.1.2.6.19.2.2.7.1.1.38,string'
) #@@CMD_PERMISION snmp protocol execution
resultSetList = resultSetList.asTable()
tcpUtil = tcpdbutils.TcpDbUtils(Framework)
regExp = '(\d+\.\d+\.\d+\.\d+)\.(\d+)\.(\d+\.\d+\.\d+\.\d+)\.(\d+)'
for resultSet in resultSetList:
try:
currRowData = string.strip(resultSet[0])
processName = string.strip(resultSet[1])
processOSH = modeling.createProcessOSH(processName, hostOSH)
OSHVResult.add(processOSH)
resArray = re.search(regExp, currRowData)
if resArray:
ip1 = resArray.group(1)
port1 = resArray.group(2)
ip2 = resArray.group(3)
port2 = resArray.group(4)
processName = resultSet[1]
# Create
# Discard invalid lines (No port#)
if port1 == '0':
continue
# Loop back and listen
if netutils.isLocalIp(ip1):
prot = 6 #tcp protocol
tcpUtil.addPortToProcess(ip_address, int(port1), -1, 1,
prot)
continue
tcpUtil.addTcpConnection(ip1, int(port1), ip2, int(port2))
tcpUtil.addTcpConnection(ip2, int(port2), ip1, int(port1))
# print '--------------------------------'
# print 'ip1 :' , ip1
# print 'port1 :' , port1
# print 'ip2 :' , ip2
# print 'port2 :' , port2
# print 'processName :' , processName
# print '--------------------------------'
except:
logger.warnException('Failed ip: %s' % (ip_address))
try:
tcpUtil.flushPortToProcesses()
except:
pass
try:
tcpUtil.flushTcpConnection()
except:
pass
tcpUtil.close()
logger.debug('Finished to process results')
def doQuerySNMPProcesses(client,
OSHVResult,
hostID,
Framework,
pid2Process=None):
processList = []
data_name_mib = '1.3.6.1.2.1.25.4.2.1.1,1.3.6.1.2.1.25.4.2.1.2,string,1.3.6.1.2.1.25.4.2.1.4,string,1.3.6.1.2.1.25.4.2.1.5,string,1.3.6.1.2.1.25.5.1.1.2,string,1.3.6.1.2.1.25.5.1.1.1,string,1.3.6.1.2.1.25.4.2.1.2,string'
resultSet = client.executeQuery(
data_name_mib) #@@CMD_PERMISION snmp protocol execution
count = 0
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
hostOSH = None
processesDoNotMatch = 0
while resultSet.next():
data_name = resultSet.getString(7)
if (data_name is None) or (data_name.find('<defunct>') > -1):
continue
#to prevent junk like '.' or something else: we assume that process name should contain at list one word character
if re.search('\w+', data_name):
if ((data_name in processList) != 0):
continue
process_pid = resultSet.getInt(2)
process_path = resultSet.getString(3)
process_path = fixProcessPath(process_path, data_name)
cmdLine = None
process_parameters = None
#NOTE: code above is commented since SNMP data is not consistent with WMI and shell so we abandon
# If one uses only SNMP - he is more than welcome to uncomment this code
# try:
# process_parameters = resultSet.getString(4)
# except:
# pass
# if (process_path != None) and (len(process_path) > 0):
# processPathStr = String(process_path)
# if processPathStr.endsWith('/') or processPathStr.endsWith('\\'):
# process_path = process_path + data_name
# processPathStr = String(process_path)
#
# cmdLine = process_path
# if (process_parameters != None) and (len(process_parameters) > 0):
# cmdLine = cmdLine + ' ' + process_parameters
pdu.addProcess(hostID, data_name, process_pid, cmdLine,
process_path, process_parameters)
count = count + 1
processList.append(data_name)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString(
'host', hostID)
processOsh = modeling.createProcessOSH(
data_name, hostOSH, cmdLine, process_pid, process_path,
process_parameters)
OSHVResult.add(processOsh)
else:
processesDoNotMatch = 1
if processesDoNotMatch:
logger.debug(
"Found processes which names do not fits pattern '\w+'")
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
logger.debug("Discovered ", str(count), " processes")
def doQueryMainframeTCP(client, OSHVResult, Framework):
ip_address = client.getIpAddress()
hostOSH = modeling.createHostOSH(client.getIpAddress())
modeling.setHostOsFamily(hostOSH, 'mainframe')
################ Query and data ################################
resultSetList = client.executeQuery('1.3.6.1.4.1.2.6.19.2.2.7.1.1.37,1.3.6.1.4.1.2.6.19.2.2.7.1.1.38,string')#@@CMD_PERMISION snmp protocol execution
resultSetList = resultSetList.asTable()
tcpUtil = tcpdbutils.TcpDbUtils(Framework)
regExp = '(\d+\.\d+\.\d+\.\d+)\.(\d+)\.(\d+\.\d+\.\d+\.\d+)\.(\d+)'
for resultSet in resultSetList:
try:
currRowData = string.strip( resultSet[0] )
processName = string.strip( resultSet[1] )
processOSH = modeling.createProcessOSH(processName, hostOSH)
OSHVResult.add(processOSH)
resArray = re.search(regExp, currRowData)
if resArray:
ip1 = resArray.group(1)
port1 = resArray.group(2)
ip2 = resArray.group(3)
port2 = resArray.group(4)
processName = resultSet[1]
# Create
# Discard invalid lines (No port#)
if port1 == '0':
continue
# Loop back and listen
if netutils.isLocalIp(ip1):
prot = 6 #tcp protocol
tcpUtil.addPortToProcess(ip_address, int(port1), -1, 1, prot)
continue
tcpUtil.addTcpConnection(ip1, int(port1), ip2, int(port2))
tcpUtil.addTcpConnection(ip2, int(port2), ip1, int(port1))
# print '--------------------------------'
# print 'ip1 :' , ip1
# print 'port1 :' , port1
# print 'ip2 :' , ip2
# print 'port2 :' , port2
# print 'processName :' , processName
# print '--------------------------------'
except:
logger.warnException('Failed ip: %s' % (ip_address))
try:
tcpUtil.flushPortToProcesses()
except:
pass
try:
tcpUtil.flushTcpConnection()
except:
pass
tcpUtil.close()
logger.debug('Finished to process results')
def buildOsh(self, hostOsh):
procPid = None
if self.pid != 0:
procPid = self.pid
return modeling.createProcessOSH(self.name, hostOsh, self.cmdline, procPid, self.path, self.params, self.owner, self.startuptime)
def doQuerySNMPProcesses(client, OSHVResult, hostID, Framework, pid2Process = None):
processList = []
data_name_mib = '1.3.6.1.2.1.25.4.2.1.1,1.3.6.1.2.1.25.4.2.1.2,string,1.3.6.1.2.1.25.4.2.1.4,string,1.3.6.1.2.1.25.4.2.1.5,string,1.3.6.1.2.1.25.5.1.1.2,string,1.3.6.1.2.1.25.5.1.1.1,string,1.3.6.1.2.1.25.4.2.1.2,string'
resultSet = client.executeQuery(data_name_mib)#@@CMD_PERMISION snmp protocol execution
count = 0
pdu = None
try:
pdu = processdbutils.ProcessDbUtils(Framework)
hostOSH = None
processesDoNotMatch = 0
while resultSet.next():
data_name = resultSet.getString(7)
if (data_name is None) or (data_name.find('<defunct>') > -1):
continue
#to prevent junk like '.' or something else: we assume that process name should contain at list one word character
if re.search('\w+', data_name):
if ((data_name in processList) != 0):
continue
process_pid = resultSet.getInt(2)
process_path = resultSet.getString(3)
process_path = fixProcessPath(process_path, data_name)
cmdLine = None
process_parameters = None
#NOTE: code above is commented since SNMP data is not consistent with WMI and shell so we abandon
# If one uses only SNMP - he is more than welcome to uncomment this code
# try:
# process_parameters = resultSet.getString(4)
# except:
# pass
# if (process_path != None) and (len(process_path) > 0):
# processPathStr = String(process_path)
# if processPathStr.endsWith('/') or processPathStr.endsWith('\\'):
# process_path = process_path + data_name
# processPathStr = String(process_path)
#
# cmdLine = process_path
# if (process_parameters != None) and (len(process_parameters) > 0):
# cmdLine = cmdLine + ' ' + process_parameters
pdu.addProcess(hostID, data_name, process_pid, cmdLine, process_path, process_parameters)
count = count + 1
processList.append(data_name)
if OSHVResult is not None:
if hostOSH == None:
hostOSH = modeling.createOshByCmdbIdString('host', hostID)
processOsh = modeling.createProcessOSH(data_name, hostOSH, cmdLine, process_pid, process_path, process_parameters)
OSHVResult.add(processOsh)
else:
processesDoNotMatch = 1
if processesDoNotMatch:
logger.debug("Found processes which names do not fits pattern '\w+'")
pdu.flushHostProcesses(hostID)
if pid2Process is not None:
pid2Process.putAll(pdu.getProcessCmdMap())
finally:
if pdu != None:
pdu.close()
logger.debug("Discovered ", str(count), " processes")