def groupServersByFullNameInCell(cell): # create catalog of all servers grouped by full name (node name + server name) serverByFullName = {} for node in cell.getNodes(): serverByFullName.update( applyMapping(jee.Server.getFullName, node.getServers())) return serverByFullName
def discover(self): ''' -> dict(int, Device), list(Connectivity) Main discovery method ''' devices = self.discoverDevices() if not devices: return devicesById = fptools.applyMapping(lambda d: d.deviceId, devices) connectivitiesByDeviceId = {} for device in devices: ports = getPortsByDevice(self.client, device) validPorts = itertools.ifilter(self._isPortValid, ports) validPortsById = fptools.applyMapping(lambda p: p.portId, validPorts) device.portsById = validPortsById modules = getModulesByDevice(self.client, device) validModules = itertools.ifilter(_isModuleValid, modules) modulesBySlot = fptools.applyMapping(lambda b: b.slot, validModules) device.modulesBySlot = modulesBySlot vlansById = self.discoverVlansByDevice(device) device.vlansById = vlansById deviceConnectivities = getLayer3TopologyByDevice( self.client, device) connectivitiesByDeviceId[device.deviceId] = deviceConnectivities if self.reportDeviceConfigs: config = self.discoverDeviceConfig(device) if config: device.config = config self.analyseDevice(device) return devicesById, connectivitiesByDeviceId
def discoverVlansByDevice(self, device): ''' Device -> dict(int, Vlan) ''' vlans = getVlansByDevice(self.client, device) validVlans = itertools.ifilter(_isVlanValid, vlans) validVlansById = fptools.applyMapping(lambda v: v.id, validVlans) return validVlansById
def discover(self): ''' -> dict(int, Device), list(Connectivity) Main discovery method ''' devices = self.discoverDevices() if not devices: return devicesById = fptools.applyMapping(lambda d: d.deviceId, devices) connectivitiesByDeviceId = {} for device in devices: ports = getPortsByDevice(self.client, device) validPorts = itertools.ifilter(self._isPortValid, ports) validPortsById = fptools.applyMapping(lambda p: p.portId, validPorts) device.portsById = validPortsById modules = getModulesByDevice(self.client, device) validModules = itertools.ifilter(_isModuleValid, modules) modulesBySlot = fptools.applyMapping(lambda b: b.slot, validModules) device.modulesBySlot = modulesBySlot vlansById = self.discoverVlansByDevice(device) device.vlansById = vlansById deviceConnectivities = getLayer3TopologyByDevice(self.client, device) connectivitiesByDeviceId[device.deviceId] = deviceConnectivities if self.reportDeviceConfigs: config = self.discoverDeviceConfig(device) if config: device.config = config self.analyseDevice(device) return devicesById, connectivitiesByDeviceId
def _discoverHostToAddressMapping(solman, reportError): r'@types: ?, (str -> None) -> dict[str, sap.Address]' logger.info("Discovery hostname to IP mapping") mapping = {} try: query = GetHostToAddressMapping() queryExecutor = TableQueryExecutor(solman) addresses = queryExecutor.executeQuery(query) getHostname = lambda a: a.hostname mapping = applyMapping(getHostname, addresses) except (Exception, JException): msg = "Failed to discovery mappings" logger.warn(msg) reportError(msg) return mapping
def _discoverSoftwareCmps(solman): '''''' logger.info("Discover Software Components") getCmpsQuery = GetComponents() executeQuery = TableQueryExecutor(solman).executeQuery sysNameToCmpPairs = executeQuery(getCmpsQuery) # get descriptions of components discoverer = SoftCmpDiscoverer(solman) localizedDescrs = discoverer.getComponentLocalizedDescriptions() isEn = SoftCmpDiscoverer.isEnglishCmpDescription descrs = ifilter(isEn, localizedDescrs) DescrClass = SoftCmpDiscoverer.ComponentLocalizedDescription descrByName = fptools.applyMapping(DescrClass.getName, descrs) r = [__fillInCmpDescription(p, descrByName) for p in sysNameToCmpPairs] logger.info("Discovered %s components" % len(r)) return r
def get_node(executor, shell_interpreter, instance_name, node_name, db2_home_path=None): r"""@types: command.CmdExecutor, shell_interpreter.Interpreter, basestring, basestring, file_topology.Path -> list(db2_model.Database) @raise ExecuteError: on DB2INSTANCE variable setting failure @raise command.ExecuteException: on db2 cmdb execution failure """ db2cmd = __get_configured_db2_cmd(shell_interpreter, instance_name, db2_home_path) list_db_directory = db2cmd.list_node_directory_show_detail() def has_node_name_and_host(node): return node.name and (node.hostname or node.is_local()) filter_invalid = command.FnCmdlet(partial(ifilter, has_node_name_and_host)) node_by_name = fptools.applyMapping(Db2.NodeEntry.name.fget, list_db_directory | executor | filter_invalid) return node_by_name.get(node_name)
def discover(self): ''' -> dict(int, Device), list(Connectivity) Main discovery method ''' devices = self.discoverDevices() if not devices: return devicesById = fptools.applyMapping(lambda d: d.deviceId, devices) allPorts = getPorts(self.client) for port in allPorts: device = devicesById.get(port.deviceId) if device and self._isPortValid(port): device.portsById[port.portId] = port modules = getModules(self.client) validModules = itertools.ifilter(_isModuleValid, modules) for module in validModules: device = devicesById.get(module.deviceId) if device is not None: device.modulesBySlot[module.slot] = module for device in devicesById.values(): vlansById = self.discoverVlansByDevice(device) device.vlansById = vlansById allConnections = getLayer3Topology(self.client) connectivitiesByDeviceId = fptools.groupby(lambda c: c.deviceId, allConnections) if self.reportDeviceConfigs: for device in devices: config = self.discoverDeviceConfig(device) if config: device.config = config for device in devices: self.analyseDevice(device) return devicesById, connectivitiesByDeviceId
def _discoverRfcDestinations(sapUtils, systemOsh, config): r'@types: SapUtils, osh, flow.DiscoveryConfigBuilder -> oshv' if not config.discoverRFCConnections: return ObjectStateHolderVector() logger.info('Discover RFC connections') getRfcCmd = sap_abap_discoverer.GetRfcDestinationsRfcCommand() connections = Sfn(getRfcCmd.getAllRfcConnections)(sapUtils) or () logger.info("Found %s possible RFC connections" % len(connections)) connections = filter(comp(sap_abap_discoverer.isEnglishVersion, third), connections) logger.info("Found %s RFC connections with EN language" % len(connections)) connByName = applyMapping(first, connections) destinations = getRfcCmd.getAllRfcDestinations(sapUtils) logger.info("Found %s RFC destinations" % len(destinations)) # get destinations with valid host destinations = [d for d in destinations if _isDestFull(d)] logger.info("Found %s destinations with host available" % len(destinations)) destinationsByHost = groupby(lambda d: d.targetHost, destinations) ips = map(Sfn(_resolve), destinationsByHost.iterkeys()) pairIpToDestinations = zip(ips, destinationsByHost.itervalues()) resolved, notResolved = partition(first, pairIpToDestinations) if notResolved: skippedDestsCount = sum([len(dests) for ip, dests in notResolved]) logger.debug("%s destinations skipped due to not resolved %s hosts" % (skippedDestsCount, len(notResolved))) vector = ObjectStateHolderVector() for ip, destinations in resolved: # TODO: # 1) query RFC connections (to get description) only for these # destinations as it will reduce amount of data fetched from system # One query for connections returns ~8K rows of data, while we are # interested in less than ~50 or even less # 2) another improvement query only records in English language countOfDests = len(destinations) host = first(destinations).targetHost reportDst = Sfn(_reportRfcDestination) logger.debug("%s destinations resolved for %s" % (countOfDests, host)) vectors = (reportDst(dst, ip, connByName, systemOsh) for dst in destinations) each(vector.addAll, ifilter(None, vectors)) return vector
def __execPropertiesCmd(self, className=None, name=None, *properties): r''' @types: str, str, dict -> object @param properties: is a variable argument sequence of tuples where first element is an property to parse and second -type of property. Type of property is represented by constant value ClusterCmd._NUMERIC_PROPERTY or ClusterCmd._STRING_PROPERTY ''' attrs = map(lambda t: t[0], properties) # create mapping of tuple to attribute name typeByAttrName = fptools.applyMapping(lambda t: t[0], properties) # validate all types assigned to properties classNamePart = (className and "%s " % className or "") namePart = (name and '"%s" ' % name or "") if self.isUsingCmd(): cmdline = '%s%s/PROP:%s' % (classNamePart, namePart, ','.join(attrs)) output = self.__exec(cmdline) else: cmdline = '%s%s | fl %s' % (classNamePart, namePart, ','.join(attrs)) output = self.__execCmdlets(cmdline) item = self._PropertiesItem() for line in output.splitlines(): for i in range(len(attrs)): attr = attrs[i] # by default all requested properties has to be set to None setattr(item, attr, None) if line.find(attr) != -1: regexp = typeByAttrName.get(attr)[1] m = re.search(regexp % attr, line) if m: setattr(item, attr, m.group(1)) # do not process this attribute any more attrs.pop(i) break return item
def get_node(executor, shell_interpreter, instance_name, node_name, db2_home_path=None): r'''@types: command.CmdExecutor, shell_interpreter.Interpreter, basestring, basestring, file_topology.Path -> list(db2_model.Database) @raise ExecuteError: on DB2INSTANCE variable setting failure @raise command.ExecuteException: on db2 cmdb execution failure ''' db2cmd = __get_configured_db2_cmd(shell_interpreter, instance_name, db2_home_path) list_db_directory = db2cmd.list_node_directory_show_detail() def has_node_name_and_host(node): return node.name and (node.hostname or node.is_local()) filter_invalid = command.FnCmdlet(partial(ifilter, has_node_name_and_host)) node_by_name = fptools.applyMapping( Db2.NodeEntry.name.fget, list_db_directory | executor | filter_invalid) return node_by_name.get(node_name)
#discover tnsnames.ora file logger.debug("try to find tnsnames.ora file") hostId = Framework.getDestinationAttribute('hostId') Framework.sendObjects( jee_discoverer.discoverTnsnamesOra(hostId, client)) # discover all processes runningProcesses = processDiscoverer.discoverAllProcesses() # make filtering for Weblogic processes serverProcesses = filter(isWeblogicAsProcess, runningProcesses) # obvious to terminate discovery job without weblogic processes running if not serverProcesses: logger.reportWarning("No Weblogic processes currently running") return ObjectStateHolderVector() # create catalog of processes by PID processByPid = applyMapping(processModule.Process.getPid, runningProcesses) # for each weblogic server process get root directory path getRootDirectoryPath = curry(getDomainRootDirPath, shell, fs, _, processByPid.get) rootDirectoryPaths = map(safeFunc(getRootDirectoryPath), serverProcesses) # group server processes by root directory path processesByRootDirPath = {} for process, path in zip(serverProcesses, rootDirectoryPaths): processesByRootDirPath.setdefault(path, []).append(process) debugGroups(processesByRootDirPath) # NEXT CASE IS NOT SUPPORTED (as property weblogic.ConfigFile # removed starting from version 9 # important to understand that some processes may specify alternative
else: #discover tnsnames.ora file logger.debug("try to find tnsnames.ora file") hostId = Framework.getDestinationAttribute('hostId') Framework.sendObjects(jee_discoverer.discoverTnsnamesOra(hostId, client)) # discover all processes runningProcesses = processDiscoverer.discoverAllProcesses() # make filtering for Weblogic processes serverProcesses = filter(isWeblogicAsProcess, runningProcesses) # obvious to terminate discovery job without weblogic processes running if not serverProcesses: logger.reportWarning("No Weblogic processes currently running") return ObjectStateHolderVector() # create catalog of processes by PID processByPid = applyMapping(processModule.Process.getPid, runningProcesses) # for each weblogic server process get root directory path getRootDirectoryPath = curry(getDomainRootDirPath, shell, fs, _, processByPid.get ) rootDirectoryPaths = map(safeFunc(getRootDirectoryPath), serverProcesses) # group server processes by root directory path processesByRootDirPath = {} for process, path in zip(serverProcesses, rootDirectoryPaths): processesByRootDirPath.setdefault(path, []).append(process) debugGroups(processesByRootDirPath) # NEXT CASE IS NOT SUPPORTED (as property weblogic.ConfigFile # removed starting from version 9 # important to understand that some processes may specify alternative
def get_network_services_by_name(network_services): return fptools.applyMapping(NetworkService.service_name.fget, network_services)
def discoverApplicationsInDomain(cell, cellLayout, fs, shell, parser, reporterCreator, jndiNamedResourceManager, sendApplicationsVector, NameBindingContent): r'@types: Domain, CellLayout, FileSystem, Shell, DescriptorParser, ReporterCreator, JndiNamedResourceManager, (ObjectStateHolderVector -> ) -> ' # create catalog of serves and cluster by full name and name accordingly serverByFullName = groupServersByFullNameInCell(cell) clusterByName = applyMapping(jee.Cluster.getName, cell.getClusters()) # discovery skeleton applicationLayout = jee_discoverer.ApplicationLayout(fs) descriptorParser = jee_discoverer.ApplicationDescriptorParser( isLoadExternalDtdEnabled()) appDiscoverer = asm_websphere_discoverer.WebsphereApplicationDiscovererByShell( shell, applicationLayout, descriptorParser) jndiNameToName = {} if NameBindingContent: logger.debug('namebinding content:', NameBindingContent.content) matches = re.findall( '<namebindings:EjbNameSpaceBinding.*?nameInNameSpace="(.*?)".*?ejbJndiName="(.*?)"', NameBindingContent.content) if matches: for match in matches: jndiNameToName[match[1]] = match[0] logger.debug('jndiNameToName: ', jndiNameToName) for server in serverByFullName.values(): # Information about deployed applications is stored # in serverindex.xml per node # Each previously discovered server may have # role of application container appServerRole = server.getRole(jee.ApplicationServerRole) # discover applications for app in (appServerRole and appServerRole.getApplications()) or (): # applications are in the cell independently of the deployment target level # cellHome/applications/<app_name|archive_name>/deployments/<module_name>/deployment.xml # if not absolute - append needed part appDeploymentDirPath = cellLayout.composeApplicationDeploymentDirPath( app.fullPath) deploymentDescriptorPath = cellLayout.composeApplicationDeploymentFilePath( app.fullPath) isAppReported = 0 vector = ObjectStateHolderVector() try: deploymentDescriptorFile = cellLayout.getFileContent( deploymentDescriptorPath) except file_topology.PathNotFoundException, pnfe: logger.warn(str(pnfe)) except (Exception, JException): logger.warn("Failed to process res file for %s" % server) else: application = appDiscoverer.discoverEarApplication( app.getName(), appDeploymentDirPath, jndiNameToName) if not application: continue try: deploymentTargetsDescriptor = parser.parseDeploymentTargets( deploymentDescriptorFile.content) except (Exception, JException): logger.warnException( "Failed to parse application deployment targets") else: applicationReporter = reporterCreator.getApplicationReporter( ) for server in deploymentTargetsDescriptor.getServers(): deploymentScope = serverByFullName.get( server.getFullName()) if deploymentScope: try: vector.addAll( applicationReporter.reportApplications( cell, deploymentScope, application)) isAppReported = 1 except Exception: logger.warnException( "Failed to report applications for the %s" % deploymentScope) for cluster in deploymentTargetsDescriptor.getClusters(): deploymentScope = clusterByName.get(cluster.getName()) if deploymentScope: try: vector.addAll( applicationReporter.reportApplications( cell, deploymentScope, application)) for node in cell.getNodes(): for server in node.getServers(): if server.hasRole( jee.ClusterMemberServerRole ) and server.getRole( jee.ClusterMemberServerRole ).clusterName == cluster.getName(): vector.addAll( applicationReporter. reportApplications( cell, server, application)) isAppReported = 1 except Exception: logger.warnException( "Failed to report applications for the %s" % deploymentScope) # report as is in scope of domain if deployment targets discovery failed if not isAppReported: try: vector.addAll( applicationReporter.reportApplications( cell, None, application)) except Exception: logger.warnException( "Failed to report applications for the %s" % cell) # report application resources for module in application.getModules(): files = filter( lambda file, expectedName=module.getDescriptorName( ): file.getName() == expectedName, module.getConfigFiles()) if files: file = files[0] try: descriptor = None if isinstance(module, jee.WebModule): descriptor = descriptorParser.parseWebModuleDescriptor( file.content, module) elif isinstance(module, jee.EjbModule): descriptor = descriptorParser.parseEjbModuleDescriptor( file.content, module) else: logger.warn( "Unknown type of JEE module: %s" % module) if descriptor: for res in descriptor.getResources(): logger.debug('resource:', res) resource = jndiNamedResourceManager.lookupResourceByJndiName( res.getName()) logger.warn("%s %s" % (resource, application)) if not (resource and resource.getOsh()): logger.warn( "%s cannot be used for %s" % (resource, application)) else: vector.addAll( applicationReporter. reportApplicationResource( application, resource)) except (Exception, JException): logger.warnException( "Failed to process %s for resources" % module) sendApplicationsVector(vector)
def groupServersByFullNameInCell(cell): # create catalog of all servers grouped by full name (node name + server name) serverByFullName = {} for node in cell.getNodes(): serverByFullName.update(applyMapping(jee.Server.getFullName, node.getServers())) return serverByFullName
def discoverApplicationsInDomain(cell, cellLayout, fs, shell, parser, reporterCreator, jndiNamedResourceManager, sendApplicationsVector, NameBindingContent): r'@types: Domain, CellLayout, FileSystem, Shell, DescriptorParser, ReporterCreator, JndiNamedResourceManager, (ObjectStateHolderVector -> ) -> ' # create catalog of serves and cluster by full name and name accordingly serverByFullName = groupServersByFullNameInCell(cell) clusterByName = applyMapping(jee.Cluster.getName, cell.getClusters()) # discovery skeleton applicationLayout = jee_discoverer.ApplicationLayout(fs) descriptorParser = jee_discoverer.ApplicationDescriptorParser(isLoadExternalDtdEnabled()) appDiscoverer = asm_websphere_discoverer.WebsphereApplicationDiscovererByShell(shell, applicationLayout, descriptorParser) jndiNameToName = {} if NameBindingContent: logger.debug('namebinding content:',NameBindingContent.content) matches = re.findall('<namebindings:EjbNameSpaceBinding.*?nameInNameSpace="(.*?)".*?ejbJndiName="(.*?)"' , NameBindingContent.content) if matches: for match in matches: jndiNameToName[match[1]]=match[0] logger.debug('jndiNameToName: ', jndiNameToName) for server in serverByFullName.values(): # Information about deployed applications is stored # in serverindex.xml per node # Each previously discovered server may have # role of application container appServerRole = server.getRole(jee.ApplicationServerRole) # discover applications for app in (appServerRole and appServerRole.getApplications()) or (): # applications are in the cell independently of the deployment target level # cellHome/applications/<app_name|archive_name>/deployments/<module_name>/deployment.xml # if not absolute - append needed part appDeploymentDirPath = cellLayout.composeApplicationDeploymentDirPath(app.fullPath) deploymentDescriptorPath = cellLayout.composeApplicationDeploymentFilePath(app.fullPath) isAppReported = 0 vector = ObjectStateHolderVector() try: deploymentDescriptorFile = cellLayout.getFileContent(deploymentDescriptorPath) except file_topology.PathNotFoundException, pnfe: logger.warn(str(pnfe)) except (Exception, JException): logger.warn("Failed to process res file for %s" % server) else: application = appDiscoverer.discoverEarApplication(app.getName(), appDeploymentDirPath, jndiNameToName) if not application: continue try: deploymentTargetsDescriptor = parser.parseDeploymentTargets(deploymentDescriptorFile.content) except (Exception, JException): logger.warnException("Failed to parse application deployment targets") else: applicationReporter = reporterCreator.getApplicationReporter() for server in deploymentTargetsDescriptor.getServers(): deploymentScope = serverByFullName.get(server.getFullName()) if deploymentScope: try: vector.addAll(applicationReporter.reportApplications(cell, deploymentScope, application)) isAppReported = 1 except Exception: logger.warnException("Failed to report applications for the %s" % deploymentScope) for cluster in deploymentTargetsDescriptor.getClusters(): deploymentScope = clusterByName.get(cluster.getName()) if deploymentScope: try: vector.addAll(applicationReporter.reportApplications(cell, deploymentScope, application)) for node in cell.getNodes(): for server in node.getServers(): if server.hasRole(jee.ClusterMemberServerRole) and server.getRole(jee.ClusterMemberServerRole).clusterName == cluster.getName(): vector.addAll(applicationReporter.reportApplications(cell, server, application)) isAppReported = 1 except Exception: logger.warnException("Failed to report applications for the %s" % deploymentScope) # report as is in scope of domain if deployment targets discovery failed if not isAppReported: try: vector.addAll(applicationReporter.reportApplications(cell, None, application)) except Exception: logger.warnException("Failed to report applications for the %s" % cell) # report application resources for module in application.getModules(): files = filter(lambda file, expectedName = module.getDescriptorName(): file.getName() == expectedName, module.getConfigFiles()) if files: file = files[0] try: descriptor = None if isinstance(module, jee.WebModule): descriptor = descriptorParser.parseWebModuleDescriptor(file.content, module) elif isinstance(module, jee.EjbModule): descriptor = descriptorParser.parseEjbModuleDescriptor(file.content, module) else: logger.warn("Unknown type of JEE module: %s" % module) if descriptor: for res in descriptor.getResources(): logger.debug('resource:',res) resource = jndiNamedResourceManager.lookupResourceByJndiName(res.getName()) logger.warn("%s %s" % (resource, application) ) if not (resource and resource.getOsh()): logger.warn("%s cannot be used for %s" % (resource, application) ) else: vector.addAll(applicationReporter.reportApplicationResource(application, resource)) except (Exception, JException): logger.warnException("Failed to process %s for resources" % module) sendApplicationsVector(vector)
debugGroupping(runtimesByNodeName) # remember administrative server if found administrativeServer = None for nodeName, nodeRuntimes in runtimesByNodeName.items(): logger.info("Discover node: %s" % nodeName) nodeLayout = websphere_discoverer.NodeLayout( cellLayout.composeNodeHomePath(nodeName), fs) node = discoverNode(nodeLayout, pathUtil) cell.addNode(node) logger.info("Discover servers") servers = parser.parseServersInServerIndex( nodeLayout.getFileContent(nodeLayout.getServerIndexPath()).content ) nodeRuntimesByServerName = applyMapping( websphere_discoverer.ServerRuntime.getServerName, nodeRuntimes ) # add to the node only running servers that match their runtime for server in servers: serverRuntime = nodeRuntimesByServerName.get(server.getName()) if serverRuntime or server.hasRole(jee.AdminServerRole): logger.info("\tResolved running %s" % server) server.nodeName = nodeName node.addServer(server) if server.hasRole(jee.AdminServerRole): administrativeServer = server logger.info("\tAdministrative server found") if serverRuntime: # assign platform version productInfo = productInfoByInstallDirPath.get( serverRuntime.findInstallRootDirPath())
def _getSystemPairsGroupedByName(sysToOshPairs): return applyMapping(comp(sap.System.getName, first), sysToOshPairs)
debugGroupping(runtimesByNodeName) # remember administrative server if found administrativeServer = None for nodeName, nodeRuntimes in runtimesByNodeName.items(): logger.info("Discover node: %s" % nodeName) nodeLayout = websphere_discoverer.NodeLayout( cellLayout.composeNodeHomePath(nodeName), fs) node = discoverNode(nodeLayout, pathUtil) cell.addNode(node) logger.info("Discover servers") servers = parser.parseServersInServerIndex( nodeLayout.getFileContent( nodeLayout.getServerIndexPath()).content) nodeRuntimesByServerName = applyMapping( websphere_discoverer.ServerRuntime.getServerName, nodeRuntimes) # add to the node only running servers that match their runtime for server in servers: serverRuntime = nodeRuntimesByServerName.get( server.getName()) if serverRuntime or server.hasRole( jee.AdminServerRole): logger.info("\tResolved running %s" % server) server.nodeName = nodeName node.addServer(server) if server.hasRole(jee.AdminServerRole): administrativeServer = server logger.info("\tAdministrative server found") if serverRuntime: # assign platform version