def _discoverServedSystems(instPf, shell, pathtools):
logger.debug("Instance name: %s" % instPf.getInstance().getName())
# we have to consider three ways how to configure served system:
# 1) for one served SAP system using 'rdisp/mshost' and 'wdisp/server_info_location'
# 2) for one or more served systems using 'wdisp/system_<xx>'
# 3) for one or more served systems using 'wdisp/server_info_location'
# when first approach used for configuration there is not
# SID for served system - so it represented as Unknown SAP systems
servedSystems = []
for servedSystem in instPf.getServedSystems():
# work with metadata-sources of different type
# - with end-point (message server)
# - pointing to server-info file
sources = servedSystem.getMetadataSources()
_, srcsWithoutEndps = partition(isSourceWithEndpoint, sources)
# ServedSystem DO covers first two approaches of configuration
# third one is configured in separate file (info.icr)
# file also contains information about application servers
# there are two ways how to configure using info.icr
# a) specifying application servers without their instance information
# b) specifying application server in scope of some instance
# a) - this applied to the case when instance information or
# information about served system known in instance profile
# b) - applied when instance profile does not have nor
# message server information nor info about served system
# x) get information about served systems from siloc-sources
# siloc-source file contains information about at least one
# served system and corresponding application servers
# possible cases:
# - unknown sap system + app servers
# - list of known sap systems + corresponding application servers
systemsFromSrcFile = []
for src in srcsWithoutEndps:
systems = _discoverServedSystemsFromSiloc(shell, pathtools, src)
systemsFromSrcFile.extend(systems)
if not srcsWithoutEndps:
# in case if served system does not have icr file
servedSystems.append(servedSystem)
elif (not isinstance(servedSystem.system, sap_webdisp.UnknownSystem)
# if known information about served system but info.icr
# bring additional info about application servers
and len(systemsFromSrcFile) == 1):
appServerEndpoints = systemsFromSrcFile[
0].getApplicationServersEndpoints()
_sys = sap_webdisp.ServedSystem(
servedSystem.system, servedSystem.getMetadataSources(),
servedSystem.getExternalServersEndpoints(),
servedSystem.getDispatchOptions(), appServerEndpoints)
servedSystems.append(_sys)
else:
servedSystems.extend(systemsFromSrcFile)
return servedSystems
def DiscoveryMain(framework, creds_manager, cred_id):
'@types: RichFramework, CredsManager, str -> list[osh]'
config = (ovm_flow.DiscoveryConfigBuilder(framework)
# parameters
.bool_params(reportStoppedVMs=False)
.int_params(commandExecutionDurationInMs=2000)
# destination data
.dest_data_params_as_int(protocol_port=None)
.dest_data_params_as_str(hostId=None)).build()
attr_name = Protocol.PROTOCOL_ATTRIBUTE_PORT
port = int(config.protocol_port
or creds_manager.get_attribute(cred_id, attr_name))
host_id = config.hostId
oshs = []
warnings = []
with closing(_createSshClient(framework, cred_id, port)) as client:
execute = _get_initialized_execute_fn(
client, config.commandExecutionDurationInMs)
server_configs = _discover_servers(execute)
vm_configs, msgs_ = _discover_vms(execute)
warnings.extend(msgs_)
mgr_version = ovm_cli.get_version(client)
logger.info("Report topology")
if not config.reportStoppedVMs:
is_running = lambda c: ovm_cli.is_vm_running(c.vm.status)
running, stopped = fptools.partition(is_running, vm_configs)
get_vm = ovm_cli.ShowVmCmd.Config.vm.fget
info_on_stopped = '\n'.join(imap(comp(str, get_vm), stopped))
logger.info("Stopped VMs are not reported: %s" % info_on_stopped)
vm_configs = running
oshs.extend(report(host_id, mgr_version, server_configs, vm_configs))
return oshs, warnings
def _drop(msg, predicate, seq):
''' Drop elements from seq that doesn't satisfy predicate with logging
message and count of skipped elements
@types: str, (A -> bool), seq[A] -> list[A]'''
with_, without = partition(predicate, seq)
if without:
logger.warn("Drop %s. %s" % (msg, len(without)))
return with_
def discover_profiles(sap_utils):
'''
@return: default profile pair (path and content) and list of pairs of
instance profiles
@types: SapUtils -> tuple[tuple[str, str]?, list[tuple[str, str]]]
'''
try:
profiles = get_profiles(sap_utils)
default_pfs, other_pfs = partition(is_default_pf, profiles)
return first(default_pfs), other_pfs
except (Exception, JException), e:
logger.warnException("Failed to discover profiles. %s" % e)
def discover_profiles(sap_utils):
'''
@return: default profile pair (path and content) and list of pairs of
instance profiles
@types: SapUtils -> tuple[tuple[str, str]?, list[tuple[str, str]]]
'''
try:
profiles = get_profiles(sap_utils)
default_pfs, other_pfs = partition(is_default_pf, profiles)
return first(default_pfs), other_pfs
except (Exception, JException), e:
logger.warnException("Failed to discover profiles. %s" % e)
def _reportSystems(servedSystems,
processOsh,
shell,
connectionIp,
applicationOsh=None):
resolveEndpoint = _getEndpResolveFn(shell, connectionIp)
vector = ObjectStateHolderVector()
softwareBuilder = sap.SoftwareBuilder()
softwareReporter = sap.SoftwareReporter(softwareBuilder)
linkR = sap.LinkReporter()
#x) report details of served systems and relation with web-dispatcher
for servedSystem in servedSystems:
# report endpoints of external servers
endpoints = servedSystem.getExternalServersEndpoints()
endpoints = flatten(keep(resolveEndpoint, endpoints))
results = (_reportServerEndp(e, processOsh, applicationOsh)
for e in endpoints)
vector.addAll(map(third, results))
# report message server endpoint (metadata-source)
# and application servers of served system
sources = servedSystem.getMetadataSources()
msgSources, sources = partition(isMessageServerSource, sources)
logger.debug("Report %s msg sources" % len(msgSources))
logger.debug("Report %s other sources" % len(sources))
# process message server, report message server
endpoints = keep(sap_webdisp.HasEndpoint.getEndpoint, msgSources)
msgBuilder = sap.MessageServerBuilder()
msgReporter = sap.CentralComponentReporter(msgBuilder)
for e in flatten(keep(resolveEndpoint, endpoints)):
e = netutils.createTcpEndpoint(e.getAddress(), e.getPort())
endpOsh, hostOsh, eVector = _reportServerEndp(e, processOsh)
vector.addAll(eVector)
msgOsh = msgReporter.reportAnonymous(hostOsh)
vector.add(msgOsh)
vector.add(linkR.reportUsage(msgOsh, endpOsh))
# process non message server sources
sources = ifilter(isSourceWithEndpoint, sources)
endpoints = keep(sap_webdisp.HasEndpoint.getEndpoint, sources)
endpoints.extend(servedSystem.getApplicationServersEndpoints())
endpoints = flatten(keep(resolveEndpoint, endpoints))
for result in (_reportServerEndp(e, processOsh) for e in endpoints):
endpOsh, hostOsh, eVector = result
vector.addAll(eVector)
appServerOsh = softwareReporter.reportUknownSoftware(hostOsh)
vector.add(appServerOsh)
vector.add(linkR.reportUsage(appServerOsh, endpOsh))
return vector
def _reportSystems(servedSystems, processOsh, shell, connectionIp, applicationOsh=None):
resolveEndpoint = _getEndpResolveFn(shell, connectionIp)
vector = ObjectStateHolderVector()
softwareBuilder = sap.SoftwareBuilder()
softwareReporter = sap.SoftwareReporter(softwareBuilder)
linkR = sap.LinkReporter()
#x) report details of served systems and relation with web-dispatcher
for servedSystem in servedSystems:
# report endpoints of external servers
endpoints = servedSystem.getExternalServersEndpoints()
endpoints = flatten(keep(resolveEndpoint, endpoints))
results = (_reportServerEndp(e, processOsh, applicationOsh) for e in endpoints)
vector.addAll(map(third, results))
# report message server endpoint (metadata-source)
# and application servers of served system
sources = servedSystem.getMetadataSources()
msgSources, sources = partition(isMessageServerSource, sources)
logger.debug("Report %s msg sources" % len(msgSources))
logger.debug("Report %s other sources" % len(sources))
# process message server, report message server
endpoints = keep(sap_webdisp.HasEndpoint.getEndpoint, msgSources)
msgBuilder = sap.MessageServerBuilder()
msgReporter = sap.CentralComponentReporter(msgBuilder)
for e in flatten(keep(resolveEndpoint, endpoints)):
e = netutils.createTcpEndpoint(e.getAddress(), e.getPort())
endpOsh, hostOsh, eVector = _reportServerEndp(e, processOsh)
vector.addAll(eVector)
msgOsh = msgReporter.reportAnonymous(hostOsh)
vector.add(msgOsh)
vector.add(linkR.reportUsage(msgOsh, endpOsh))
# process non message server sources
sources = ifilter(isSourceWithEndpoint, sources)
endpoints = keep(sap_webdisp.HasEndpoint.getEndpoint, sources)
endpoints.extend(servedSystem.getApplicationServersEndpoints())
endpoints = flatten(keep(resolveEndpoint, endpoints))
for result in (_reportServerEndp(e, processOsh) for e in endpoints):
endpOsh, hostOsh, eVector = result
vector.addAll(eVector)
appServerOsh = softwareReporter.reportUknownSoftware(hostOsh)
vector.add(appServerOsh)
vector.add(linkR.reportUsage(appServerOsh, endpOsh))
return vector
def _discoverRfcDestinations(sapUtils, systemOsh, config):
r'@types: SapUtils, osh, flow.DiscoveryConfigBuilder -> oshv'
if not config.discoverRFCConnections:
return ObjectStateHolderVector()
logger.info('Discover RFC connections')
getRfcCmd = sap_abap_discoverer.GetRfcDestinationsRfcCommand()
connections = Sfn(getRfcCmd.getAllRfcConnections)(sapUtils) or ()
logger.info("Found %s possible RFC connections" % len(connections))
connections = filter(comp(sap_abap_discoverer.isEnglishVersion, third),
connections)
logger.info("Found %s RFC connections with EN language" % len(connections))
connByName = applyMapping(first, connections)
destinations = getRfcCmd.getAllRfcDestinations(sapUtils)
logger.info("Found %s RFC destinations" % len(destinations))
# get destinations with valid host
destinations = [d for d in destinations if _isDestFull(d)]
logger.info("Found %s destinations with host available" % len(destinations))
destinationsByHost = groupby(lambda d: d.targetHost, destinations)
ips = map(Sfn(_resolve), destinationsByHost.iterkeys())
pairIpToDestinations = zip(ips, destinationsByHost.itervalues())
resolved, notResolved = partition(first, pairIpToDestinations)
if notResolved:
skippedDestsCount = sum([len(dests) for ip, dests in notResolved])
logger.debug("%s destinations skipped due to not resolved %s hosts" %
(skippedDestsCount, len(notResolved)))
vector = ObjectStateHolderVector()
for ip, destinations in resolved:
# TODO:
# 1) query RFC connections (to get description) only for these
# destinations as it will reduce amount of data fetched from system
# One query for connections returns ~8K rows of data, while we are
# interested in less than ~50 or even less
# 2) another improvement query only records in English language
countOfDests = len(destinations)
host = first(destinations).targetHost
reportDst = Sfn(_reportRfcDestination)
logger.debug("%s destinations resolved for %s" % (countOfDests, host))
vectors = (reportDst(dst, ip, connByName, systemOsh) for dst in destinations)
each(vector.addAll, ifilter(None, vectors))
return vector
def get_applicable_credentials(framework, portPattern):
r'@types: Framework, str -> iterator[tuple[int, str]]'
logger.debug('Getting applicable credentials to createClient')
credIds = _get_sap_java_creds(framework)
# get ports available on destination
destPorts = framework.getTriggerCIDataAsList('sap_jmx_port') or ()
destPorts = set(ifilter(portPattern.match, destPorts))
portAttr = Protocol.PROTOCOL_ATTRIBUTE_PORT
ports = [framework.getProtocolProperty(id_, portAttr) for id_ in credIds]
# get SAP java credentials availabe in ucmdb
# get as list of pairs (port, credential ID)
portToCredId = izip(ports, credIds)
# separate all credentials onto two groups - with port defined and without
withPort, withoutPort = partition(first, portToCredId)
return chain(
# add credentials only if configured there port exists in dst
((c, dp) for _, c in withoutPort for dp in destPorts),
# create variations of credentials with all destination ports
((c, p) for p, c in withPort if p in destPorts))
def discoverAllInstancesByNamesOnly(client):
r''' Can be only applied for discovery by JMX due to deserialization
limitatations of WebServices client
@types: BaseSapJmxClient, DiscoveryConfig -> tuple[oshv, tuple[str]]'''
discoverer = sap_jee_discoverer.ClusterDiscoverer(client)
cluster, instanceNames = discoverer.getClusterDetails()
parseInst = Sf(_createAnonymousInstFromFullName)
insts = keep(parseInst, instanceNames)
system = sap.System(cluster.getName())
systemOsh, clusterOsh, vector = _reportSapSystem(system)
ipsPerInst = zip(map(_resolveInstHostname, insts), insts)
resolved, notResolved = partition(first, ipsPerInst)
warnings = ()
if notResolved:
warnings = ("Some instances are not reported "
"due to unresolved address",)
vectors = (third(reportInst(i, system, systemOsh, clusterOsh, ips))
for ips, i in resolved)
each(vector.addAll, vectors)
return vector, warnings
def DiscoveryMain(framework, creds_manager, cred_id):
'@types: RichFramework, CredsManager, str -> list[osh]'
config = (
ovm_flow.DiscoveryConfigBuilder(framework)
# parameters
.bool_params(reportStoppedVMs=False).int_params(
commandExecutionDurationInMs=2000)
# destination data
.dest_data_params_as_int(protocol_port=None).dest_data_params_as_str(
hostId=None)).build()
attr_name = Protocol.PROTOCOL_ATTRIBUTE_PORT
port = int(config.protocol_port
or creds_manager.get_attribute(cred_id, attr_name))
host_id = config.hostId
oshs = []
warnings = []
with closing(_createSshClient(framework, cred_id, port)) as client:
execute = _get_initialized_execute_fn(
client, config.commandExecutionDurationInMs)
server_configs = _discover_servers(execute)
vm_configs, msgs_ = _discover_vms(execute)
warnings.extend(msgs_)
mgr_version = ovm_cli.get_version(client)
logger.info("Report topology")
if not config.reportStoppedVMs:
is_running = lambda c: ovm_cli.is_vm_running(c.vm.status)
running, stopped = fptools.partition(is_running, vm_configs)
get_vm = ovm_cli.ShowVmCmd.Config.vm.fget
info_on_stopped = '\n'.join(imap(comp(str, get_vm), stopped))
logger.info("Stopped VMs are not reported: %s" % info_on_stopped)
vm_configs = running
oshs.extend(report(host_id, mgr_version, server_configs, vm_configs))
return oshs, warnings
def _discoverServedSystems(instPf, shell, pathtools):
logger.debug("Instance name: %s" % instPf.getInstance().getName())
# we have to consider three ways how to configure served system:
# 1) for one served SAP system using 'rdisp/mshost' and 'wdisp/server_info_location'
# 2) for one or more served systems using 'wdisp/system_<xx>'
# 3) for one or more served systems using 'wdisp/server_info_location'
# when first approach used for configuration there is not
# SID for served system - so it represented as Unknown SAP systems
servedSystems = []
for servedSystem in instPf.getServedSystems():
# work with metadata-sources of different type
# - with end-point (message server)
# - pointing to server-info file
sources = servedSystem.getMetadataSources()
_, srcsWithoutEndps = partition(isSourceWithEndpoint, sources)
# ServedSystem DO covers first two approaches of configuration
# third one is configured in separate file (info.icr)
# file also contains information about application servers
# there are two ways how to configure using info.icr
# a) specifying application servers without their instance information
# b) specifying application server in scope of some instance
# a) - this applied to the case when instance information or
# information about served system known in instance profile
# b) - applied when instance profile does not have nor
# message server information nor info about served system
# x) get information about served systems from siloc-sources
# siloc-source file contains information about at least one
# served system and corresponding application servers
# possible cases:
# - unknown sap system + app servers
# - list of known sap systems + corresponding application servers
systemsFromSrcFile = []
for src in srcsWithoutEndps:
systems = _discoverServedSystemsFromSiloc(shell, pathtools, src)
systemsFromSrcFile.extend(systems)
if not srcsWithoutEndps:
# in case if served system does not have icr file
servedSystems.append(servedSystem)
elif (not isinstance(servedSystem.system, sap_webdisp.UnknownSystem)
# if known information about served system but info.icr
# bring additional info about application servers
and len(systemsFromSrcFile) == 1):
appServerEndpoints = systemsFromSrcFile[0].getApplicationServersEndpoints()
_sys = sap_webdisp.ServedSystem(
servedSystem.system,
servedSystem.getMetadataSources(),
servedSystem.getExternalServersEndpoints(),
servedSystem.getDispatchOptions(),
appServerEndpoints)
servedSystems.append(_sys)
else:
servedSystems.extend(systemsFromSrcFile)
return servedSystems
def reportTrexHostNode(hostNode, topology, isBiaProduct):
r'@types: TrexTopologyConfig.HostNode, TrexTopologyConfig, bool -> ObjectStateHolderVector'
trexBuilder = sap_trex.Builder()
trexReporter = sap_trex.Reporter(trexBuilder)
hostReporter = sap_trex.HostReporter(sap_trex.HostBuilder())
endpointReporter = netutils.EndpointReporter(
netutils.ServiceEndpointBuilder())
linkReporter = sap.LinkReporter()
softwareBuilder = sap.SoftwareBuilder()
# x) create sap system
system = hostNode.system
vector = ObjectStateHolderVector()
# process NameServer endpoints and ignore loopback endpoints
isLoopbackEndpoint = lambda e: netutils.isLoopbackIp(e.getAddress())
_, endpoints = fptools.partition(isLoopbackEndpoint,
hostNode.nameServerEndpoints)
# x) create host OSH
hostOsh = hostReporter.reportHostByHostname(hostNode.name)
vector.add(hostOsh)
# x) report IPs
ips = map(netutils.Endpoint.getAddress, endpoints)
ipOshs = map(modeling.createIpOSH, ips)
fptools.each(vector.add, ipOshs)
#vector.addAll(ipOshs)
# x) report containment between host nad ips
reportContainment = fptools.partiallyApply(linkReporter.reportContainment,
hostOsh, fptools._)
fptools.each(vector.add, map(reportContainment, ipOshs))
# x) report end-points
reportEndpoint = fptools.partiallyApply(endpointReporter.reportEndpoint,
fptools._, hostOsh)
endpointOshs = map(reportEndpoint, endpoints)
fptools.each(vector.add, endpointOshs)
# x) report TREX instance itself
instanceOsh = trexReporter.reportInstance(first(system.getInstances()),
hostOsh)
# x) mark as BIA or plain-TREX
productName = (isBiaProduct and sap_trex.Product.BIA.instanceProductName
or sap_trex.Product.TREX.instanceProductName)
softwareBuilder.updateDiscoveredProductName(instanceOsh, productName)
# x) set name server role (master, slave or 1st master)
nameServerPort = first(endpoints).getPort()
nameServerEndpoint = netutils.createTcpEndpoint(hostNode.name,
nameServerPort)
topologyGlobals = topology.getGlobals()
isMaster = nameServerEndpoint in (fptools.safeFunc(
topologyGlobals.getMasterEndpoints)() or ())
isActiveMaster = nameServerEndpoint in (fptools.safeFunc(
topologyGlobals.getActiveMasterEndpoints)() or ())
trexBuilder.updateNameServerMode(
instanceOsh,
(isMaster and
(isActiveMaster and sap_trex.NameServerMode.FIRST_MASTER
or sap_trex.NameServerMode.MASTER) or sap_trex.NameServerMode.SLAVE))
vector.add(instanceOsh)
# x) DO NOT report 'membership' between system and instance
# Explanation:
# sometimes you can discover systems that don't have relationship to current host.
# This can lead to incorrect merging of to systems (inside OSH vector)
# systemOsh = trexReporter.reportSystem(system)
# vector.add(systemOsh)
# vector.add(linkReporter.reportMembership(systemOsh, instanceOsh))
# x) report 'usage' between instance and endpoints of name-server
reportUsage = fptools.partiallyApply(linkReporter.reportUsage, instanceOsh,
fptools._)
fptools.each(vector.add, map(reportUsage, endpointOshs))
return vector
def reportTrexHostNode(hostNode, topology, isBiaProduct):
r'@types: TrexTopologyConfig.HostNode, TrexTopologyConfig, bool -> ObjectStateHolderVector'
trexBuilder = sap_trex.Builder()
trexReporter = sap_trex.Reporter(trexBuilder)
hostReporter = sap_trex.HostReporter(sap_trex.HostBuilder())
endpointReporter = netutils.EndpointReporter(netutils.ServiceEndpointBuilder())
linkReporter = sap.LinkReporter()
softwareBuilder = sap.SoftwareBuilder()
# x) create sap system
system = hostNode.system
vector = ObjectStateHolderVector()
# process NameServer endpoints and ignore loopback endpoints
isLoopbackEndpoint = lambda e: netutils.isLoopbackIp(e.getAddress())
_, endpoints = fptools.partition(isLoopbackEndpoint,
hostNode.nameServerEndpoints)
# x) create host OSH
hostOsh = hostReporter.reportHostByHostname(hostNode.name)
vector.add(hostOsh)
# x) report IPs
ips = map(netutils.Endpoint.getAddress, endpoints)
ipOshs = map(modeling.createIpOSH, ips)
fptools.each(vector.add, ipOshs)
#vector.addAll(ipOshs)
# x) report containment between host nad ips
reportContainment = fptools.partiallyApply(linkReporter.reportContainment, hostOsh, fptools._)
fptools.each(vector.add, map(reportContainment, ipOshs))
# x) report end-points
reportEndpoint = fptools.partiallyApply(endpointReporter.reportEndpoint, fptools._, hostOsh)
endpointOshs = map(reportEndpoint, endpoints)
fptools.each(vector.add, endpointOshs)
# x) report TREX instance itself
instanceOsh = trexReporter.reportInstance(first(system.getInstances()), hostOsh)
# x) mark as BIA or plain-TREX
productName = (isBiaProduct
and sap_trex.Product.BIA.instanceProductName
or sap_trex.Product.TREX.instanceProductName)
softwareBuilder.updateDiscoveredProductName(instanceOsh, productName)
# x) set name server role (master, slave or 1st master)
nameServerPort = first(endpoints).getPort()
nameServerEndpoint = netutils.createTcpEndpoint(hostNode.name, nameServerPort)
topologyGlobals = topology.getGlobals()
isMaster = nameServerEndpoint in (
fptools.safeFunc(topologyGlobals.getMasterEndpoints)() or ()
)
isActiveMaster = nameServerEndpoint in (
fptools.safeFunc(topologyGlobals.getActiveMasterEndpoints)() or ()
)
trexBuilder.updateNameServerMode( instanceOsh,
(isMaster
and (isActiveMaster
and sap_trex.NameServerMode.FIRST_MASTER
or sap_trex.NameServerMode.MASTER)
or sap_trex.NameServerMode.SLAVE))
vector.add(instanceOsh)
# x) DO NOT report 'membership' between system and instance
# Explanation:
# sometimes you can discover systems that don't have relationship to current host.
# This can lead to incorrect merging of to systems (inside OSH vector)
# systemOsh = trexReporter.reportSystem(system)
# vector.add(systemOsh)
# vector.add(linkReporter.reportMembership(systemOsh, instanceOsh))
# x) report 'usage' between instance and endpoints of name-server
reportUsage = fptools.partiallyApply(linkReporter.reportUsage, instanceOsh, fptools._)
fptools.each(vector.add, map(reportUsage, endpointOshs))
return vector