def __findLocation(self, hsplSet, landscape):
"""
Finds a suitable plug-in and location for the HSPL refinement.
@param hsplSet: The HSPL set to use.
@param landscape: The landscape.
@return: The plug-in and IT resource identifier, or [None, None] if nobody is useful. What a shame.
"""
plugins = set()
identifiers = set()
hsplAction = hsplSet.findtext("{%s}hspl/{%s}action" %
(getHSPLNamespace(), getHSPLNamespace()))
for i in self.pluginManager.getPluginsOfCategory("Action"):
pluginAction = i.details.get("Core", "Action")
pluginCapabilities = set(
re.split("\s*,\s*", i.details.get("Core", "Capabilities")))
if hsplAction == pluginAction:
for identifier, capabilities in landscape.items():
if pluginCapabilities.issubset(capabilities):
plugins.add(i)
identifiers.add(identifier)
# Picks a random plug-in.
if len(plugins) == 0:
plugin = None
else:
plugin = random.sample(plugins, 1)[0]
# Picks a random identifier.
if len(identifiers) == 0:
identifier = None
else:
identifier = random.sample(identifiers, 1)[0]
return [plugin, identifier]
def __getHash(self, hspl):
"""
Retrieves the constant hash of an HSPL.
@param hspls: The HSPL.
@return: The constant hash of the HSPL.
"""
subject = hspl.find("{%s}subject" % getHSPLNamespace())
action = hspl.find("{%s}action" % getHSPLNamespace())
trafficConstraints = hspl.find("{%s}traffic-constraints" %
getHSPLNamespace())
h = 1
h = 37 * h + hash(etree.tostring(subject))
h = 37 * h + hash(etree.tostring(action))
h = 37 * h + hash(etree.tostring(trafficConstraints))
return h
def remove(self, hspl):
"""
Removes an HSPL from the map.
@param hspl: The HSPL to remove.
"""
hsplObject = hspl.findtext("{%s}object" % getHSPLNamespace())
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?", hsplObject)
if m:
key = self.__getHash(hspl)
address = ip_network(m.group(1))
port = m.group(4)
if port == "any":
port = "*"
prefixLength = address.prefixlen
number = int(address.network_address)
mapPrefixes = self.__map[key]
for i in range(0, prefixLength + 1):
if i in mapPrefixes:
mapAddresses = mapPrefixes[i]
n = (number >> (32 - i)) << (32 - i)
if n in mapAddresses:
mapPort = mapAddresses[n]
if port in mapPort:
mapPort[port].remove(hspl)
if port != "*" and "*" in mapPort:
mapPort["*"].remove(hspl)
if hspl in self.__hspls:
self.__hspls.remove(hspl)
def find(self, hspl, forcePrefixLength=None, forceAnyPort=False):
"""
Finds all the inclusions of a HSPLs.
@param hspl: The HSPL to search for the inclusions.
@param forcePrefixLength: The prefix length to use for the search or None to use the HSPL prefix length.
@param forceAnyPort: A value stating if we want to force an any port address or keep the original value.
@return: The set of HSPLs included by the passed HSPL.
"""
inclusions = set()
hsplObject = hspl.findtext("{%s}object" % getHSPLNamespace())
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?", hsplObject)
if m:
key = self.__getHash(hspl)
address = ip_network(m.group(1))
port = m.group(4)
if port == "any" or forceAnyPort:
port = "*"
if forcePrefixLength is not None:
prefixLength = forcePrefixLength
else:
prefixLength = address.prefixlen
number = int(address.network_address)
if key in self.__map:
mapPrefixes = self.__map[key]
mapAddresses = mapPrefixes[prefixLength]
n = (number >> (32 - prefixLength)) << (32 - prefixLength)
if n in mapAddresses:
mapPort = mapAddresses[n]
if port in mapPort:
inclusions.update(mapPort[port])
return inclusions
def add(self, hspl):
"""
Adds a new HSPL to the map.
@param hspl: The HSPL to add.
"""
hsplObject = hspl.findtext("{%s}object" % getHSPLNamespace())
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?", hsplObject)
if m:
key = self.__getHash(hspl)
address = ip_network(m.group(1))
port = m.group(4)
prefixLength = address.prefixlen
number = int(address.network_address)
if key not in self.__map:
self.__map[key] = {}
mapPrefixes = self.__map[key]
for i in range(0, prefixLength + 1):
if i not in mapPrefixes:
mapPrefixes[i] = {}
mapAddresses = mapPrefixes[i]
n = (number >> (32 - i)) << (32 - i)
if n not in mapAddresses:
mapAddresses[n] = {}
mapPort = mapAddresses[n]
if "*" not in mapPort:
mapPort["*"] = set()
mapPort["*"].add(hspl)
if port not in mapPort:
mapPort[port] = set()
mapPort[port].add(hspl)
self.__hspls.add(hspl)
def __cleanAndMerge(self, recommendations):
"""
Polish an HSPL set by removing the duplicate HSPLs and merging them together, if needed. We only work on the objects.
@param recommendations: The HSPL recommendations set to use.
@return: The cleaned HSPL set.
"""
hsplMergeInclusions = int(
self.configParser.getboolean("global", "hsplMergeInclusions"))
hsplMergeWithAnyPorts = int(
self.configParser.getboolean("global", "hsplMergeWithAnyPorts"))
hsplMergeWithSubnets = int(
self.configParser.getboolean("global", "hsplMergeWithSubnets"))
if not hsplMergeInclusions and not hsplMergeWithAnyPorts and not hsplMergeWithSubnets:
return recommendations
count = 0
for hsplSet in recommendations:
# Pass 0: create the map.
hsplMap = HSPLMap()
for i in hsplSet:
if i.tag == "{%s}hspl" % getHSPLNamespace():
hsplMap.add(i)
# Pass 1: removes the included HSPLs.
if hsplMergeInclusions:
includedHSPLs = self.__mergeInclusions(hsplSet, hsplMap)
if includedHSPLs > 1:
LOG.debug("%d included HSPLs removed for the HSPL set %d.",
includedHSPLs, count)
else:
LOG.debug("%d included HSPL removed for the HSPL set %d.",
includedHSPLs, count)
# Pass 2: merges the IP address using * as the port number.
if hsplMergeWithAnyPorts:
mergedHSPLs = self.__mergeWithAnyPorts(hsplSet, hsplMap)
if mergedHSPLs > 1:
LOG.debug(
"%d HSPLs merged using any ports for the HSPL set %d.",
mergedHSPLs, count)
else:
LOG.debug(
"%d HSPL merged using any port for the HSPL set %d.",
mergedHSPLs, count)
# Pass 3: merges the HSPLs, if needed.
if hsplMergeWithSubnets:
mergedHSPLs = self.__mergeWithSubnets(hsplSet, hsplMap)
if mergedHSPLs > 1:
LOG.debug(
"%d HSPLs merged using subnets for the HSPL set %d.",
mergedHSPLs, count)
else:
LOG.debug(
"%d HSPL merged using subnets for the HSPL set %d.",
mergedHSPLs, count)
return recommendations
def _doObjectTest(self, attackFile, landscapeFile, maximumHSPLs,
expectedObjects):
"""
Tests the HSPL generation.
@param attackFile: The attack file to read.
@param maximumHSPLs: The maximum number of HSPLs.
@param landscapeFile: The CSF file to read.
@param expectedObjects: The list of expected objects.
"""
cyberTop = CyberTop(getTestFilePath("cybertop.cfg"),
getTestFilePath("logging.ini"))
r = cyberTop.getMSPLsFromFile(getTestFilePath(attackFile),
getTestFilePath(landscapeFile))
self.assertIsNotNone(r)
[recommendation, _] = r
for hsplSet in recommendation:
objects = hsplSet.findall("{%s}hspl/{%s}object" %
(getHSPLNamespace(), getHSPLNamespace()))
self.assertLessEqual(len(objects), maximumHSPLs)
for i in range(len(objects)):
self.assertIn(objects[i].text, expectedObjects)
def __mergeWithSubnets(self, hsplSet, hsplMap):
"""
Merges together several HSPLs by using subnets.
@param hsplSet: The HSPL set to edit.
@return: The number of merged HSPLs removed.
"""
hsplMergingThreshold = int(
self.configParser.get("global", "hsplMergingThreshold"))
hsplMergingMinBits = int(
self.configParser.get("global", "hsplMergingMinBits"))
hsplMergingMaxBits = int(
self.configParser.get("global", "hsplMergingMaxBits"))
bits = hsplMergingMinBits
merged = set()
while len(hsplMap.getHSPLs()
) > hsplMergingThreshold and bits >= hsplMergingMaxBits:
hspls = set()
mergedHSPLs = []
for i in hsplMap.getHSPLs():
if i not in hspls:
inclusions = hsplMap.find(i, bits, True)
if len(inclusions) > 1:
mergedHSPLs.append(inclusions)
hspls.update(inclusions)
for i in mergedHSPLs:
s = set(i)
first = s.pop()
firstObject = first.find("{%s}object" % getHSPLNamespace())
m = re.match("((\d+\.\d+\.\d+\.\d+)(/\d+)?)(:(\d+|\*|any))?",
firstObject.text)
address = ip_address(m.group(2))
number = int(address)
n = (number >> (32 - bits)) << (32 - bits)
firstObject.text = "%s/%d:*" % (ip_address(n), bits)
for j in s:
hsplMap.remove(j)
hsplSet.remove(j)
merged.update(s)
bits -= 1
return len(merged)
def __checkIncludedHSPLs(self, hspl1, hspl2):
"""
Checks if the first HSPLs includes the second one.
@param hspl1: The first HSPL.
@param hspl2: The second HSPL.
@return: True if the two HSPLs are equivalent or the first HSPL include the second one, False otherwise.
"""
subject1 = hspl1.findtext("{%s}subject" % getHSPLNamespace())
subject2 = hspl2.findtext("{%s}subject" % getHSPLNamespace())
action1 = hspl1.findtext("{%s}action" % getHSPLNamespace())
action2 = hspl2.findtext("{%s}action" % getHSPLNamespace())
object1 = hspl1.findtext("{%s}object" % getHSPLNamespace())
object2 = hspl2.findtext("{%s}object" % getHSPLNamespace())
trafficConstraints1 = hspl1.find("{%s}traffic-constraints" %
getHSPLNamespace())
trafficConstraints2 = hspl2.find("{%s}traffic-constraints" %
getHSPLNamespace())
m1 = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?", object1)
m2 = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?", object2)
objectCheck = False
if m1 and m2:
address1 = ip_network(m1.group(1))
address2 = ip_network(m2.group(1))
n1 = int(address1.network_address) >> (32 - address1.prefixlen)
n2 = int(address2.network_address) >> (32 - address1.prefixlen)
port1 = m1.group(4)
port2 = m2.group(4)
if n1 == n2 and (port1 == port2 or port1 == "*" or port1 == "any"):
objectCheck = True
if subject1 == subject2 and action1 == action2 and objectCheck and self.__checkEqualXML(
trafficConstraints1, trafficConstraints2):
return True
return False
def __mergeWithAnyPorts(self, hsplSet, hsplMap):
"""
Merges together several HSPLs by using * as a port.
@param hsplSet: The HSPL set to edit.
@return: The number of merged HSPLs removed.
"""
hsplMergingThreshold = int(
self.configParser.get("global", "hsplMergingThreshold"))
if len(hsplSet) <= hsplMergingThreshold:
return 0
hspls = set()
mergedHSPLs = []
for i in hsplMap.getHSPLs():
if i not in hspls:
inclusions = hsplMap.find(i, None, True)
if len(inclusions) > 1:
mergedHSPLs.append(inclusions)
hspls.update(inclusions)
for i in mergedHSPLs:
s = set(i)
first = s.pop()
firstObject = first.find("{%s}object" % getHSPLNamespace())
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
firstObject.text)
address = m.group(1)
firstObject.text = "%s:*" % address
for j in s:
hsplMap.remove(j)
if j in hsplSet:
hsplSet.remove(j)
return len(hspls) - len(mergedHSPLs)
def _doHSPLTest(self,
attackFile,
landscapeFile,
expectedCount,
expectedProtocols,
expectedActions,
expectedSubjects=None,
expectedObjectPorts=None):
"""
Tests the HSPL generation.
@param attackFile: The attack file to read.
@param landscapeFile: The CSF file to read.
@param expectedCount: The number of expected recommendation.
@param expectedProtocols: The expected protocol list.
@param expectedActions: The expected actions list.
@param expectedSubjects: The expected subject list or None if this test must be skipped.
@param expectedObjectPorts: The expected object port list or None if this test must be skipped.
"""
cyberTop = CyberTop(getTestFilePath("cybertop.cfg"),
getTestFilePath("logging.ini"))
r = cyberTop.getMSPLsFromFile(getTestFilePath(attackFile),
getTestFilePath(landscapeFile))
self.assertIsNotNone(r)
[recommendation, _] = r
self.assertEqual(expectedCount, len(recommendation))
for hsplSet in recommendation:
good = True
protocols = hsplSet.findall(
"{%s}hspl/{%s}traffic-constraints/{%s}type" %
(getHSPLNamespace(), getHSPLNamespace(), getHSPLNamespace()))
if len(protocols) != len(expectedProtocols):
good = False
continue
p1 = []
for i in protocols:
p1.append(i.text)
p1 = sorted(p1)
p2 = sorted(expectedProtocols)
for i in range(len(protocols)):
if p1[i] != p2[i]:
good = False
actions = hsplSet.findall("{%s}hspl/{%s}action" %
(getHSPLNamespace(), getHSPLNamespace()))
objects = hsplSet.findall("{%s}hspl/{%s}object" %
(getHSPLNamespace(), getHSPLNamespace()))
subjects = hsplSet.findall(
"{%s}hspl/{%s}subject" %
(getHSPLNamespace(), getHSPLNamespace()))
self.assertEqual(len(actions), len(expectedActions))
for i in range(len(actions)):
if actions[i].text != expectedActions[i]:
good = False
if expectedSubjects is not None:
if len(subjects) != len(expectedSubjects):
good = False
for i in range(0, len(subjects)):
if subjects[i].text != expectedSubjects[i]:
good = False
if expectedObjectPorts is not None:
if len(objects) != len(expectedObjectPorts):
good = False
for i in range(0, len(objects)):
parts = objects[i].text.split(":")
if len(parts) != 2:
good = False
if parts[1] != expectedObjectPorts[i]:
good = False
if good:
return
self.fail("No HSPL set respect the expectations!")
def getMSPLs(self, hsplRecommendations, landscape, anomaly_name):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param recommendations: The HSPL recommendations to use.
@param landscape: The landscape.
@return: The XML MSPL set that can mitigate the attack. It is None if no HSPL is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if hsplRecommendations is None:
return None
schema = etree.XMLSchema(etree.parse(getMSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
for hsplSet in hsplRecommendations:
msplSet = etree.SubElement(recommendations,
"{%s}mspl-set" % getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
msplSeverity = hsplSet.findtext(
"{%s}context/{%s}severity" %
(getHSPLNamespace(), getHSPLNamespace()))
msplType = hsplSet.findtext(
"{%s}context/{%s}type" %
(getHSPLNamespace(), getHSPLNamespace()))
msplTimestamp = hsplSet.findtext(
"{%s}context/{%s}timestamp" %
(getHSPLNamespace(), getHSPLNamespace()))
# Adds the context.
context = etree.SubElement(msplSet,
"{%s}context" % getMSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getMSPLNamespace()).text = msplSeverity
#etree.SubElement(context, "{%s}type" % getMSPLNamespace()).text = msplType
etree.SubElement(context, "{%s}type" %
getMSPLNamespace()).text = anomaly_name
etree.SubElement(context, "{%s}timestamp" %
getMSPLNamespace()).text = msplTimestamp
# Finds a plug-in that can create a configured IT resource.
[plugin, identifier] = self.__findLocation(hsplSet, landscape)
plugin.plugin_object.setup(self.configParser)
LOG.info("Check if VNSFO API call (experimental) is enabled")
# Check if VNSFO is integrated to recommendations engine
vnsfo_integration = self.configParser.getboolean(
"vnsfo", "enable_vnsfo_api_call")
if vnsfo_integration:
# Creates the IT resource.
LOG.info("Experimental: contact vNSFO API")
vnsfo_base_url = self.configParser.get("vnsfo",
"vnsfo_base_url")
vnsfo_timeout = int(
self.configParser.get("vnsfo", "vnsfo_timeout"))
if not vnsfo_base_url:
LOG.info("VNSFO base URL empty. Fallback to stable.")
else:
LOG.info("Retrieving VNSF running ID for: " + identifier)
vnfr_id = retrieve_vnsfr_id(vnsfo_base_url, identifier,
anomaly_name, vnsfo_timeout)
if vnfr_id:
LOG.info("VNSF running ID is: " + vnfr_id)
identifier = vnfr_id
else:
LOG.info("Stable solution selected.")
itResource = etree.SubElement(
msplSet, "{%s}it-resource" % getMSPLNamespace(),
{"id": identifier})
# Calls the plug-in to configure the IT resource.
plugin.plugin_object.configureITResource(itResource, hsplSet)
if schema.validate(recommendations):
LOG.debug(
etree.tostring(recommendations, pretty_print=True).decode())
return recommendations
else:
LOG.critical("Invalid MSPL recommendations generated.")
raise SyntaxError("Invalid MSPL recommendations generated.")
def getHSPLs(self, attack, recipes, landscape):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param attack: The attack to mitigate.
@param recipes: The recipes to use.
@param landscape: The landscape.
@return: The XML HSPL set that can mitigate the attack. It is None if no recipe is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if recipes is None:
return None
schema = etree.XMLSchema(etree.parse(getHSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
for recipe in recipes:
hsplSet = etree.SubElement(recommendations,
"{%s}hspl-set" % getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
recipeName = recipe.findtext("{%s}name" % getRecipeNamespace())
recipeAction = recipe.findtext("{%s}action" % getRecipeNamespace())
recipeSubjectAnyAddress = recipe.findtext(
"{%s}subject-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeSubjectAnyPort = recipe.findtext(
"{%s}subject-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyAddress = recipe.findtext(
"{%s}object-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyPort = recipe.findtext(
"{%s}object-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeType = recipe.findtext(
"{%s}traffic-constraints/{%s}type" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeMaxConnections = recipe.findtext(
"{%s}traffic-constraints/{%s}max-connections" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeRateLimit = recipe.findtext(
"{%s}traffic-constraints/{%s}rate-limit" %
(getRecipeNamespace(), getRecipeNamespace()))
# Adds the context.
context = etree.SubElement(hsplSet,
"{%s}context" % getHSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getHSPLNamespace()).text = str(attack.severity)
etree.SubElement(context, "{%s}type" %
getHSPLNamespace()).text = attack.type
etree.SubElement(
context, "{%s}timestamp" %
getHSPLNamespace()).text = attack.getTimestamp().isoformat()
# Filters the events.
events = []
recipeFilters = recipe.find("{%s}filters" % getRecipeNamespace())
evaluation = "or"
if recipeFilters is None:
events = attack.events
else:
if "evaluation" in recipeFilters.attrib.keys():
evaluation = recipeFilters.attrib["evaluation"]
for i in attack.events:
if evaluation == "or":
test = False
else:
test = True
for j in self.pluginManager.getPluginsOfCategory("Filter"):
pluginTag = j.details.get("Core", "Tag")
filterValues = recipeFilters.findall(
"{%s}%s" % (getRecipeNamespace(), pluginTag))
for k in filterValues:
t = j.plugin_object.filter(k.text, i)
if evaluation == "or":
test = test or t
else:
test = test and t
if not test:
events.append(i)
# Adds an HSPL for each event.
count = 0
for i in events:
count += 1
hspl = etree.SubElement(hsplSet,
"{%s}hspl" % getHSPLNamespace())
etree.SubElement(
hspl, "{%s}name" %
getHSPLNamespace()).text = "%s #%d" % (recipeName, count)
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.target)
targetAddress = m.group(1)
targetPort = m.group(4)
if recipeSubjectAnyAddress is not None:
targetAddress = "*"
if recipeSubjectAnyPort is not None:
targetPort = "*"
etree.SubElement(
hspl, "{%s}subject" %
getHSPLNamespace()).text = "%s:%s" % (targetAddress,
targetPort)
etree.SubElement(hspl, "{%s}action" %
getHSPLNamespace()).text = recipeAction
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.attacker)
attackerAddress = m.group(1)
attackerPort = m.group(4)
if recipeObjectAnyAddress is not None:
attackerAddress = "*"
if recipeObjectAnyPort is not None:
attackerPort = "*"
etree.SubElement(
hspl, "{%s}object" %
getHSPLNamespace()).text = "%s:%s" % (attackerAddress,
attackerPort)
trafficConstraints = etree.SubElement(
hspl, "{%s}traffic-constraints" % getHSPLNamespace())
if recipeType is not None:
eventType = recipeType
else:
eventType = i.fields["protocol"]
etree.SubElement(trafficConstraints, "{%s}type" %
getHSPLNamespace()).text = eventType
if eventType == "TCP" and recipeMaxConnections is not None:
etree.SubElement(
trafficConstraints, "{%s}max-connections" %
getHSPLNamespace()).text = recipeMaxConnections
if recipeRateLimit is not None:
etree.SubElement(trafficConstraints, "{%s}rate-limit" %
getHSPLNamespace()).text = recipeRateLimit
LOG.debug(etree.tostring(recommendations, pretty_print=True).decode())
if schema.validate(recommendations):
return self.__cleanAndMerge(recommendations)
else:
LOG.critical("Invalid HSPL recommendations generated.")
raise SyntaxError("Invalid HSPL recommendations generated.")
