def process_IN_CLOSE_WRITE(self, event):
"""
Handles a file creation.
@param event: The file event.
"""
LOG.debug("Callback from event in directory")
try:
# First, translate the CSV in HSPL, MSPL sets
[hsplSet, msplSet] = self.getMSPLsFromFile(
event.pathname, self.configParser.get("global",
"landscapeFile"))
# Then, if extra logging is activated, print HSPL (and/or MSPL)
# to an external file
if self.configParser.has_option("global", "hsplsFile"):
with open(self.configParser.get("global", "hsplsFile"),
"w") as f:
f.write(
etree.tostring(hsplSet, pretty_print=True).decode())
if self.configParser.has_option("global", "msplsFile"):
with open(self.configParser.get("global", "msplsFile"),
"w") as f:
f.write(
etree.tostring(msplSet, pretty_print=True).decode())
# Finally, sends everything to RabbitMQ.
self.send(hsplSet, msplSet)
except BaseException as e:
LOG.critical(str(e))
if self.channel is not None:
if not self.channel.is_closed:
self.channel.close()
def getLandscape(self, fileName):
"""
Creates a landscape map by parsing an XML file.
@param fileName: the file name of the XML file to parse.
@return: the landscape map.
@raise IOError: if the file has an invalid format.
"""
schema = etree.XMLSchema(etree.parse(getLandscapeXSDFile()))
parser = etree.XMLParser(schema = schema)
if not os.path.exists(fileName):
LOG.critical("The file '%s' does not exist", fileName)
raise IOError("The file '%s' does not exist", fileName)
root = etree.parse(fileName, parser).getroot()
landscape = {}
for i in root:
identifier = i.attrib["id"]
capabilities = set()
for j in i.findall("{%s}capability" % getLandscapeNamespace()):
capabilities.add(j.text)
landscape[identifier] = capabilities
LOG.info("Landscape with %d IT resources read.", len(landscape))
return landscape
def parse(self, fileName, count, line):
"""
Parses an event line.
@param fileName: The current file name or None if this is a list.
@param count: The current line count.
@param line: The line to parse.
@return: The attack event or None if this line should be silently ignored.
@raise IOError: if the line contains something invalid.
"""
if re.match("\s*#.*", line):
return None
parts = re.split("\s*,\s*|\s+", line.rstrip())
if parts == [""]:
return None
try:
timestamp = parser.parse("%s %s" % (parts[0], parts[1]))
sourceAddress = ipaddress.ip_address(parts[9])
destinationAddress = ipaddress.ip_address(parts[10])
destinationPort = int(parts[11])
sourcePort = int(parts[12])
protocol = parts[13]
inputPackets = int(parts[14])
inputBytes = int(parts[15])
outputPackets = int(parts[16])
outputBytes = int(parts[17])
attackEvent = AttackEvent(
timestamp, "%s:%d" % (sourceAddress, sourcePort),
"%s:%d" % (destinationAddress, destinationPort))
attackEvent.fields["protocol"] = protocol
attackEvent.fields["inputPackets"] = inputPackets
attackEvent.fields["inputBytes"] = inputBytes
attackEvent.fields["outputPackets"] = outputPackets
attackEvent.fields["outputBytes"] = outputBytes
return attackEvent
except:
if count == 1:
return None
elif fileName is None:
LOG.critical("The line %d has an invalid format.", count)
raise IOError("The line %d has an invalid format." % count)
else:
LOG.critical(
"The line %d in the file '%s' has an invalid format.",
count, fileName)
raise IOError(
"The line %d in the file '%s' has an invalid format." %
(count, fileName))
def parse(self, fileName, count, line):
"""
Parses an event line.
@param fileName: The current file name or None if this is a list.
@param count: The current line count.
@param line: The line to parse.
@return: The attack event or None if this line should be silently ignored.
@raise IOError: if the line contains something invalid.
"""
if re.match("\s*#.*", line):
return None
parts = re.split("\s*,\s*|\s+", line.rstrip())
if parts == [""]:
return None
try:
timestamp = parser.parse(
"%s %s %s %s" %
(parts[0], parts[1], parts[2], parts[3].split('.')[0]))
frameLength = int(parts[6])
destinationAddress = ipaddress.ip_address(parts[7])
query = parts[8]
queryClass = int(parts[9], 16)
queryType = int(parts[10])
queryResponseCode = int(parts[11])
attackEvent = AttackEvent(timestamp, "%s:*" % destinationAddress,
"0.0.0.0/0:53")
attackEvent.fields["frameLength"] = frameLength
attackEvent.fields["query"] = query
attackEvent.fields["queryClass"] = queryClass
attackEvent.fields["queryType"] = queryType
attackEvent.fields["queryResponseCode"] = queryResponseCode
return attackEvent
except:
if count == 1:
return None
elif fileName is None:
LOG.critical("The line %d has an invalid format.", count)
raise IOError("The line %d has an invalid format." % count)
else:
LOG.critical(
"The line %d in the file '%s' has an invalid format.",
count, fileName)
raise IOError(
"The line %d in the file '%s' has an invalid format." %
(count, fileName))
def getAttackFromList(self, identifier, severity, attackType, attackList, anomaly_name):
"""
Creates an attack object by parsing a CSV list.
@param identifier: the attack id.
@param severity: the attack severity.
@param attackType: the attack type.
@param attackList: the list to parse.
@return: the attack object.
@raise IOError: if the file has an invalid format or if no suitable parser plug-in is available.
"""
# Finds a suitable parser.
plugin = None
for i in self.pluginManager.getPluginsOfCategory("Parser"):
pluginFileName = i.details.get("Core", "FileName")
if re.match(pluginFileName, attackType):
plugin = i
break
if plugin is None:
LOG.critical("No suitable attack event parser found.")
raise IOError("No suitable attack event parser found")
# Creates an attack object.
attackType = plugin.details.get("Core", "Attack")
attack = Attack(severity, attackType, identifier, anomaly_name)
# Opens the file and read the events.
count = 0
for line in attackList:
count += 1
event = plugin.plugin_object.parse(None, count, line)
print(">>>>>>", event)
if event is not None:
attack.events.append(event)
# Third: checks if there are some events.
if count == 0:
LOG.critical("The list is empty")
raise IOError("The list is empty")
LOG.info("Parsed an attack of type '%s' with severity %d and containing %d events.", attack.type, attack.severity, len(attack.events))
return attack
def retrieve_vnsfr_id(vnsfo_base_url, vnfd_id, attack_name, timeout):
LOG.info("Request vNSFO API call for vnsfd_id=" + vnfd_id +
" and attack type=" + attack_name)
url = vnsfo_base_url + "/vnsf/r4/running"
LOG.info("VNSFO API call: " + url)
try:
response = requests.get(url, verify=False, timeout=timeout)
LOG.info("VNSFO API response: " + response.text)
vnsfs = response.json()["vnsf"]
# search for first running instance which matches the query
for vnsf in vnsfs:
target_vnf = vnsf['vnfd_id'][:-5].lower()
if vnfd_id[:-5].lower() in target_vnf and attack_name.lower(
) in target_vnf:
LOG.info("Found instance=" + vnsf['vnfr_id'] + " for attack=" +
attack_name)
return vnsf['vnfr_id']
LOG.info("No running instance found from VNSFO API.")
return None
except Exception as e:
LOG.critical("VNSFO API error: " + str(e))
return None
def getMSPLs(self, hsplRecommendations, landscape, anomaly_name):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param recommendations: The HSPL recommendations to use.
@param landscape: The landscape.
@return: The XML MSPL set that can mitigate the attack. It is None if no HSPL is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if hsplRecommendations is None:
return None
schema = etree.XMLSchema(etree.parse(getMSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
for hsplSet in hsplRecommendations:
msplSet = etree.SubElement(recommendations,
"{%s}mspl-set" % getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
msplSeverity = hsplSet.findtext(
"{%s}context/{%s}severity" %
(getHSPLNamespace(), getHSPLNamespace()))
msplType = hsplSet.findtext(
"{%s}context/{%s}type" %
(getHSPLNamespace(), getHSPLNamespace()))
msplTimestamp = hsplSet.findtext(
"{%s}context/{%s}timestamp" %
(getHSPLNamespace(), getHSPLNamespace()))
# Adds the context.
context = etree.SubElement(msplSet,
"{%s}context" % getMSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getMSPLNamespace()).text = msplSeverity
#etree.SubElement(context, "{%s}type" % getMSPLNamespace()).text = msplType
etree.SubElement(context, "{%s}type" %
getMSPLNamespace()).text = anomaly_name
etree.SubElement(context, "{%s}timestamp" %
getMSPLNamespace()).text = msplTimestamp
# Finds a plug-in that can create a configured IT resource.
[plugin, identifier] = self.__findLocation(hsplSet, landscape)
plugin.plugin_object.setup(self.configParser)
LOG.info("Check if VNSFO API call (experimental) is enabled")
# Check if VNSFO is integrated to recommendations engine
vnsfo_integration = self.configParser.getboolean(
"vnsfo", "enable_vnsfo_api_call")
if vnsfo_integration:
# Creates the IT resource.
LOG.info("Experimental: contact vNSFO API")
vnsfo_base_url = self.configParser.get("vnsfo",
"vnsfo_base_url")
vnsfo_timeout = int(
self.configParser.get("vnsfo", "vnsfo_timeout"))
if not vnsfo_base_url:
LOG.info("VNSFO base URL empty. Fallback to stable.")
else:
LOG.info("Retrieving VNSF running ID for: " + identifier)
vnfr_id = retrieve_vnsfr_id(vnsfo_base_url, identifier,
anomaly_name, vnsfo_timeout)
if vnfr_id:
LOG.info("VNSF running ID is: " + vnfr_id)
identifier = vnfr_id
else:
LOG.info("Stable solution selected.")
itResource = etree.SubElement(
msplSet, "{%s}it-resource" % getMSPLNamespace(),
{"id": identifier})
# Calls the plug-in to configure the IT resource.
plugin.plugin_object.configureITResource(itResource, hsplSet)
if schema.validate(recommendations):
LOG.debug(
etree.tostring(recommendations, pretty_print=True).decode())
return recommendations
else:
LOG.critical("Invalid MSPL recommendations generated.")
raise SyntaxError("Invalid MSPL recommendations generated.")
def getAttackFromFile(self, fileName):
"""
Creates an attack object by parsing a CSV file.
@param fileName: the file name of the CSV file to parse.
@return: the attack object.
@raise IOError: if the file has an invalid format or if no suitable parser plug-in is available.
"""
# First: checks if the file is a regular file.
if not ntpath.isfile(fileName):
LOG.critical("The file '%s' is not a regular file.", fileName)
raise IOError("The file '%s' is not a regular file." % fileName)
# Second: checks the file name format.
match = re.match("^(Very Low|Very low|very low|Low|low|High|high|Very High|Very high|high)-(.+)?-(\d+)\.csv$", os.path.basename(fileName))
if match:
severity = match.group(1).lower()
if severity == "very low":
severity = 1
elif severity == "low":
severity = 2
elif severity == "high":
severity = 3
else:
severity = 4
attackType = match.group(2)
identifier = int(match.group(3))
else:
severity = 4
attackType = os.path.splitext(ntpath.basename(fileName))[0]
identifier = None
anomaly_name = attackType
# Finds a suitable parser.
plugin = None
for i in self.pluginManager.getPluginsOfCategory("Parser"):
pluginFileName = i.details.get("Core", "FileName")
if re.match(pluginFileName, attackType):
plugin = i
break
if plugin is None:
LOG.critical("No suitable attack event parser found.")
raise IOError("No suitable attack event parser found")
# Creates an attack object.
attackType = plugin.details.get("Core", "Attack")
attack = Attack(severity, attackType, identifier, anomaly_name)
# Opens the file and read the events.
count = 0
with open(fileName, "rt") as csv:
for line in csv:
count += 1
event = plugin.plugin_object.parse(fileName, count, line)
if event is not None:
attack.events.append(event)
# Third: checks if there are some events.
if count <= 1:
LOG.critical("The file '%s' is empty.", fileName)
raise IOError("The file '%s' is empty." % fileName)
LOG.info("Parsed an attack of type '%s' with severity %d and containing %d events.", attack.type, attack.severity, len(attack.events))
return attack
def __init__(self,
configurationFileName=None,
logConfigurationFileName=None):
"""
Constructor.
@param configurationFileName: the name of the configuration file to
parse.
@param logConfigurationFileName: the name of the log configuration file
to use.
"""
# Configures the logging.
log.load_settings(logConfigurationFileName)
# Configures the configuration file parser.
self.configParser = ConfigParser()
if configurationFileName is None:
c = self.configParser.read(getConfigurationFile())
else:
c = self.configParser.read(configurationFileName)
if len(c) > 0:
LOG.debug("Configuration file '%s' read." % c[0])
else:
LOG.critical("Cannot read the configuration file from '%s'." %
configurationFileName)
raise IOError("Cannot read the configuration file from '%s'" %
configurationFileName)
# Configures the plug-ins.
self.pluginManager = PluginManager()
self.pluginManager.setPluginPlaces([getPluginDirectory()])
self.pluginManager.setCategoriesFilter({
"Action": ActionPlugin,
"Parser": ParserPlugin,
"Filter": FilterPlugin
})
self.pluginManager.collectPlugins()
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Parser"))
if pluginsCount > 1:
LOG.info("Found %d attack event parser plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event parser plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Filter"))
if pluginsCount > 1:
LOG.info("Found %d attack event filter plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event filter plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Action"))
if pluginsCount > 1:
LOG.info("Found %d action plug-ins.", pluginsCount)
else:
LOG.info("Found %d action plug-in.", pluginsCount)
# Loads all the sub-modules.
self.parser = Parser(self.configParser, self.pluginManager)
self.recipesReasoner = RecipesReasoner(self.configParser,
self.pluginManager)
self.hsplReasoner = HSPLReasoner(self.configParser, self.pluginManager)
self.msplReasoner = MSPLReasoner(self.configParser, self.pluginManager)
# Starts with no attack info.
self.attacks = {}
# Connection to the DARE rabbitMQ queue
self.r_connection = None
self.r_channel = None
self.r_closingConnection = False
LOG.info("CyberSecurity Topologies initialized.")
def getHSPLs(self, attack, recipes, landscape):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param attack: The attack to mitigate.
@param recipes: The recipes to use.
@param landscape: The landscape.
@return: The XML HSPL set that can mitigate the attack. It is None if no recipe is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if recipes is None:
return None
schema = etree.XMLSchema(etree.parse(getHSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
for recipe in recipes:
hsplSet = etree.SubElement(recommendations,
"{%s}hspl-set" % getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
recipeName = recipe.findtext("{%s}name" % getRecipeNamespace())
recipeAction = recipe.findtext("{%s}action" % getRecipeNamespace())
recipeSubjectAnyAddress = recipe.findtext(
"{%s}subject-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeSubjectAnyPort = recipe.findtext(
"{%s}subject-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyAddress = recipe.findtext(
"{%s}object-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyPort = recipe.findtext(
"{%s}object-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeType = recipe.findtext(
"{%s}traffic-constraints/{%s}type" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeMaxConnections = recipe.findtext(
"{%s}traffic-constraints/{%s}max-connections" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeRateLimit = recipe.findtext(
"{%s}traffic-constraints/{%s}rate-limit" %
(getRecipeNamespace(), getRecipeNamespace()))
# Adds the context.
context = etree.SubElement(hsplSet,
"{%s}context" % getHSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getHSPLNamespace()).text = str(attack.severity)
etree.SubElement(context, "{%s}type" %
getHSPLNamespace()).text = attack.type
etree.SubElement(
context, "{%s}timestamp" %
getHSPLNamespace()).text = attack.getTimestamp().isoformat()
# Filters the events.
events = []
recipeFilters = recipe.find("{%s}filters" % getRecipeNamespace())
evaluation = "or"
if recipeFilters is None:
events = attack.events
else:
if "evaluation" in recipeFilters.attrib.keys():
evaluation = recipeFilters.attrib["evaluation"]
for i in attack.events:
if evaluation == "or":
test = False
else:
test = True
for j in self.pluginManager.getPluginsOfCategory("Filter"):
pluginTag = j.details.get("Core", "Tag")
filterValues = recipeFilters.findall(
"{%s}%s" % (getRecipeNamespace(), pluginTag))
for k in filterValues:
t = j.plugin_object.filter(k.text, i)
if evaluation == "or":
test = test or t
else:
test = test and t
if not test:
events.append(i)
# Adds an HSPL for each event.
count = 0
for i in events:
count += 1
hspl = etree.SubElement(hsplSet,
"{%s}hspl" % getHSPLNamespace())
etree.SubElement(
hspl, "{%s}name" %
getHSPLNamespace()).text = "%s #%d" % (recipeName, count)
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.target)
targetAddress = m.group(1)
targetPort = m.group(4)
if recipeSubjectAnyAddress is not None:
targetAddress = "*"
if recipeSubjectAnyPort is not None:
targetPort = "*"
etree.SubElement(
hspl, "{%s}subject" %
getHSPLNamespace()).text = "%s:%s" % (targetAddress,
targetPort)
etree.SubElement(hspl, "{%s}action" %
getHSPLNamespace()).text = recipeAction
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.attacker)
attackerAddress = m.group(1)
attackerPort = m.group(4)
if recipeObjectAnyAddress is not None:
attackerAddress = "*"
if recipeObjectAnyPort is not None:
attackerPort = "*"
etree.SubElement(
hspl, "{%s}object" %
getHSPLNamespace()).text = "%s:%s" % (attackerAddress,
attackerPort)
trafficConstraints = etree.SubElement(
hspl, "{%s}traffic-constraints" % getHSPLNamespace())
if recipeType is not None:
eventType = recipeType
else:
eventType = i.fields["protocol"]
etree.SubElement(trafficConstraints, "{%s}type" %
getHSPLNamespace()).text = eventType
if eventType == "TCP" and recipeMaxConnections is not None:
etree.SubElement(
trafficConstraints, "{%s}max-connections" %
getHSPLNamespace()).text = recipeMaxConnections
if recipeRateLimit is not None:
etree.SubElement(trafficConstraints, "{%s}rate-limit" %
getHSPLNamespace()).text = recipeRateLimit
LOG.debug(etree.tostring(recommendations, pretty_print=True).decode())
if schema.validate(recommendations):
return self.__cleanAndMerge(recommendations)
else:
LOG.critical("Invalid HSPL recommendations generated.")
raise SyntaxError("Invalid HSPL recommendations generated.")