def getLandscape(self, fileName):
"""
Creates a landscape map by parsing an XML file.
@param fileName: the file name of the XML file to parse.
@return: the landscape map.
@raise IOError: if the file has an invalid format.
"""
schema = etree.XMLSchema(etree.parse(getLandscapeXSDFile()))
parser = etree.XMLParser(schema = schema)
if not os.path.exists(fileName):
LOG.critical("The file '%s' does not exist", fileName)
raise IOError("The file '%s' does not exist", fileName)
root = etree.parse(fileName, parser).getroot()
landscape = {}
for i in root:
identifier = i.attrib["id"]
capabilities = set()
for j in i.findall("{%s}capability" % getLandscapeNamespace()):
capabilities.add(j.text)
landscape[identifier] = capabilities
LOG.info("Landscape with %d IT resources read.", len(landscape))
return landscape
def start(self):
input = self.configParser.get("global", "inputMethod")
LOG.info("Input method: " + input)
if input == "queue":
self.listenRabbitMQ()
elif input == "csv":
self.listenFolder()
elif input == "all":
self.spawnThreads()
else:
LOG.error("Unknown input method chosen (queue, csv allowed)")
return
LOG.info("Cybertop started")
def send(self, hsplSet, msplSet):
"""
Sends the policies to the dashboard.
@param hsplSet: the HSPL set.
@param msplSet: the MSPL set.
"""
if (self.configParser.has_option("global", "dashboardHost")
and self.configParser.has_option("global", "dashboardPort")
and self.configParser.has_option("global", "dashboardExchange")
and self.configParser.has_option("global", "dashboardTopic")
and self.configParser.has_option("global", "dashboardAttempts")
and self.configParser.has_option("global",
"dashboardRetryDelay")):
host = self.configParser.get("global", "dashboardHost")
port = self.configParser.getint("global", "dashboardPort")
connectionAttempts = self.configParser.getint(
"global", "dashboardAttempts")
retryDelay = self.configParser.getint("global",
"dashboardRetryDelay")
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=host,
port=port,
connection_attempts=connectionAttempts,
retry_delay=retryDelay,
blocked_connection_timeout=300))
self.channel = connection.channel()
self.channel.exchange_declare(exchange=self.configParser.get(
"global", "dashboardExchange"),
exchange_type="topic")
LOG.info("Connected to the dashboard at " + host + ":" + str(port))
hsplString = etree.tostring(hsplSet).decode()
msplString = etree.tostring(msplSet).decode()
content = self.configParser.get("global", "dashboardContent")
if content == "HSPL":
message = hsplString
elif content == "MSPL":
message = msplString
else:
message = hsplString + msplString
LOG.info("Pushing the remediation to the dashboard")
exchange = self.configParser.get("global", "dashboardExchange")
topic = self.configParser.get("global", "dashboardTopic")
self.channel.basic_publish(exchange=exchange,
routing_key=topic,
body=message)
LOG.debug("Dashboard RabbitMQ exchange: " + exchange + " topic: " +
topic)
LOG.info("Remediation forwarded to the dashboard")
self.channel.close()
LOG.info("Connection with the dashboard closed")
def getAttackFromList(self, identifier, severity, attackType, attackList, anomaly_name):
"""
Creates an attack object by parsing a CSV list.
@param identifier: the attack id.
@param severity: the attack severity.
@param attackType: the attack type.
@param attackList: the list to parse.
@return: the attack object.
@raise IOError: if the file has an invalid format or if no suitable parser plug-in is available.
"""
# Finds a suitable parser.
plugin = None
for i in self.pluginManager.getPluginsOfCategory("Parser"):
pluginFileName = i.details.get("Core", "FileName")
if re.match(pluginFileName, attackType):
plugin = i
break
if plugin is None:
LOG.critical("No suitable attack event parser found.")
raise IOError("No suitable attack event parser found")
# Creates an attack object.
attackType = plugin.details.get("Core", "Attack")
attack = Attack(severity, attackType, identifier, anomaly_name)
# Opens the file and read the events.
count = 0
for line in attackList:
count += 1
event = plugin.plugin_object.parse(None, count, line)
print(">>>>>>", event)
if event is not None:
attack.events.append(event)
# Third: checks if there are some events.
if count == 0:
LOG.critical("The list is empty")
raise IOError("The list is empty")
LOG.info("Parsed an attack of type '%s' with severity %d and containing %d events.", attack.type, attack.severity, len(attack.events))
return attack
def retrieve_vnsfr_id(vnsfo_base_url, vnfd_id, attack_name, timeout):
LOG.info("Request vNSFO API call for vnsfd_id=" + vnfd_id +
" and attack type=" + attack_name)
url = vnsfo_base_url + "/vnsf/r4/running"
LOG.info("VNSFO API call: " + url)
try:
response = requests.get(url, verify=False, timeout=timeout)
LOG.info("VNSFO API response: " + response.text)
vnsfs = response.json()["vnsf"]
# search for first running instance which matches the query
for vnsf in vnsfs:
target_vnf = vnsf['vnfd_id'][:-5].lower()
if vnfd_id[:-5].lower() in target_vnf and attack_name.lower(
) in target_vnf:
LOG.info("Found instance=" + vnsf['vnfr_id'] + " for attack=" +
attack_name)
return vnsf['vnfr_id']
LOG.info("No running instance found from VNSFO API.")
return None
except Exception as e:
LOG.critical("VNSFO API error: " + str(e))
return None
def getMSPLs(self, hsplRecommendations, landscape, anomaly_name):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param recommendations: The HSPL recommendations to use.
@param landscape: The landscape.
@return: The XML MSPL set that can mitigate the attack. It is None if no HSPL is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if hsplRecommendations is None:
return None
schema = etree.XMLSchema(etree.parse(getMSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
for hsplSet in hsplRecommendations:
msplSet = etree.SubElement(recommendations,
"{%s}mspl-set" % getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
msplSeverity = hsplSet.findtext(
"{%s}context/{%s}severity" %
(getHSPLNamespace(), getHSPLNamespace()))
msplType = hsplSet.findtext(
"{%s}context/{%s}type" %
(getHSPLNamespace(), getHSPLNamespace()))
msplTimestamp = hsplSet.findtext(
"{%s}context/{%s}timestamp" %
(getHSPLNamespace(), getHSPLNamespace()))
# Adds the context.
context = etree.SubElement(msplSet,
"{%s}context" % getMSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getMSPLNamespace()).text = msplSeverity
#etree.SubElement(context, "{%s}type" % getMSPLNamespace()).text = msplType
etree.SubElement(context, "{%s}type" %
getMSPLNamespace()).text = anomaly_name
etree.SubElement(context, "{%s}timestamp" %
getMSPLNamespace()).text = msplTimestamp
# Finds a plug-in that can create a configured IT resource.
[plugin, identifier] = self.__findLocation(hsplSet, landscape)
plugin.plugin_object.setup(self.configParser)
LOG.info("Check if VNSFO API call (experimental) is enabled")
# Check if VNSFO is integrated to recommendations engine
vnsfo_integration = self.configParser.getboolean(
"vnsfo", "enable_vnsfo_api_call")
if vnsfo_integration:
# Creates the IT resource.
LOG.info("Experimental: contact vNSFO API")
vnsfo_base_url = self.configParser.get("vnsfo",
"vnsfo_base_url")
vnsfo_timeout = int(
self.configParser.get("vnsfo", "vnsfo_timeout"))
if not vnsfo_base_url:
LOG.info("VNSFO base URL empty. Fallback to stable.")
else:
LOG.info("Retrieving VNSF running ID for: " + identifier)
vnfr_id = retrieve_vnsfr_id(vnsfo_base_url, identifier,
anomaly_name, vnsfo_timeout)
if vnfr_id:
LOG.info("VNSF running ID is: " + vnfr_id)
identifier = vnfr_id
else:
LOG.info("Stable solution selected.")
itResource = etree.SubElement(
msplSet, "{%s}it-resource" % getMSPLNamespace(),
{"id": identifier})
# Calls the plug-in to configure the IT resource.
plugin.plugin_object.configureITResource(itResource, hsplSet)
if schema.validate(recommendations):
LOG.debug(
etree.tostring(recommendations, pretty_print=True).decode())
return recommendations
else:
LOG.critical("Invalid MSPL recommendations generated.")
raise SyntaxError("Invalid MSPL recommendations generated.")
def getAttackFromFile(self, fileName):
"""
Creates an attack object by parsing a CSV file.
@param fileName: the file name of the CSV file to parse.
@return: the attack object.
@raise IOError: if the file has an invalid format or if no suitable parser plug-in is available.
"""
# First: checks if the file is a regular file.
if not ntpath.isfile(fileName):
LOG.critical("The file '%s' is not a regular file.", fileName)
raise IOError("The file '%s' is not a regular file." % fileName)
# Second: checks the file name format.
match = re.match("^(Very Low|Very low|very low|Low|low|High|high|Very High|Very high|high)-(.+)?-(\d+)\.csv$", os.path.basename(fileName))
if match:
severity = match.group(1).lower()
if severity == "very low":
severity = 1
elif severity == "low":
severity = 2
elif severity == "high":
severity = 3
else:
severity = 4
attackType = match.group(2)
identifier = int(match.group(3))
else:
severity = 4
attackType = os.path.splitext(ntpath.basename(fileName))[0]
identifier = None
anomaly_name = attackType
# Finds a suitable parser.
plugin = None
for i in self.pluginManager.getPluginsOfCategory("Parser"):
pluginFileName = i.details.get("Core", "FileName")
if re.match(pluginFileName, attackType):
plugin = i
break
if plugin is None:
LOG.critical("No suitable attack event parser found.")
raise IOError("No suitable attack event parser found")
# Creates an attack object.
attackType = plugin.details.get("Core", "Attack")
attack = Attack(severity, attackType, identifier, anomaly_name)
# Opens the file and read the events.
count = 0
with open(fileName, "rt") as csv:
for line in csv:
count += 1
event = plugin.plugin_object.parse(fileName, count, line)
if event is not None:
attack.events.append(event)
# Third: checks if there are some events.
if count <= 1:
LOG.critical("The file '%s' is empty.", fileName)
raise IOError("The file '%s' is empty." % fileName)
LOG.info("Parsed an attack of type '%s' with severity %d and containing %d events.", attack.type, attack.severity, len(attack.events))
return attack
def __init__(self,
configurationFileName=None,
logConfigurationFileName=None):
"""
Constructor.
@param configurationFileName: the name of the configuration file to
parse.
@param logConfigurationFileName: the name of the log configuration file
to use.
"""
# Configures the logging.
log.load_settings(logConfigurationFileName)
# Configures the configuration file parser.
self.configParser = ConfigParser()
if configurationFileName is None:
c = self.configParser.read(getConfigurationFile())
else:
c = self.configParser.read(configurationFileName)
if len(c) > 0:
LOG.debug("Configuration file '%s' read." % c[0])
else:
LOG.critical("Cannot read the configuration file from '%s'." %
configurationFileName)
raise IOError("Cannot read the configuration file from '%s'" %
configurationFileName)
# Configures the plug-ins.
self.pluginManager = PluginManager()
self.pluginManager.setPluginPlaces([getPluginDirectory()])
self.pluginManager.setCategoriesFilter({
"Action": ActionPlugin,
"Parser": ParserPlugin,
"Filter": FilterPlugin
})
self.pluginManager.collectPlugins()
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Parser"))
if pluginsCount > 1:
LOG.info("Found %d attack event parser plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event parser plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Filter"))
if pluginsCount > 1:
LOG.info("Found %d attack event filter plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event filter plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Action"))
if pluginsCount > 1:
LOG.info("Found %d action plug-ins.", pluginsCount)
else:
LOG.info("Found %d action plug-in.", pluginsCount)
# Loads all the sub-modules.
self.parser = Parser(self.configParser, self.pluginManager)
self.recipesReasoner = RecipesReasoner(self.configParser,
self.pluginManager)
self.hsplReasoner = HSPLReasoner(self.configParser, self.pluginManager)
self.msplReasoner = MSPLReasoner(self.configParser, self.pluginManager)
# Starts with no attack info.
self.attacks = {}
# Connection to the DARE rabbitMQ queue
self.r_connection = None
self.r_channel = None
self.r_closingConnection = False
LOG.info("CyberSecurity Topologies initialized.")
def processMessage(self, channel, method, header, body):
"""
Handles a RabbitMQ message.
@param channel: The channel.
@param method: The method.
@param header: The message header.
@param body: The message body.
"""
LOG.debug("Callback from event in RabbitMQ")
line = body.decode()
dialect = Sniffer().sniff(line)
fields = []
for i in reader([line], dialect):
fields += i
LOG.debug("DARE RabbitMQ message: " + line)
if len(fields) == 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1],
IGNORECASE) and fields[3] == "start":
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.info("Attack started (id: %d, severity: %s, type: %s)" %
(identifier, severity, attackType))
key = "%d-%s-%s" % (identifier, severity, attackType)
if key in self.attacks:
LOG.warning("Duplicate start message")
else:
self.attacks[key] = AttackInfo(identifier, severity,
attackType)
elif len(fields) == 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1],
IGNORECASE) and fields[3] == "stop":
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.info("Attack stopped (id: %d, severity: %s, type: %s)" %
(identifier, severity, attackType))
# store the anomaly detection name in a variable
anomaly_name = attackType
LOG.debug("Anomaly name is: " + anomaly_name)
key = "%d-%s-%s" % (identifier, severity, attackType)
if key not in self.attacks:
LOG.warning("Stop message without initial start message")
else:
attackInfo = self.attacks[key]
self.attacks.pop(key)
identifier = attackInfo.getIdentifier()
severity = attackInfo.getSeverity()
attackType = attackInfo.getType()
events = attackInfo.getEvents()
landscapeFileName = self.configParser.get(
"global", "landscapeFile")
LOG.debug("Get mspls from list")
# First, translate the CSV in HSPL, MSPL sets
[hsplSet,
msplSet] = self.getMSPLsFromList(identifier, severity,
attackType, events,
landscapeFileName,
anomaly_name)
LOG.debug("Got mspls from list")
# Then, if extra logging is activated, print HSPL (and/or MSPL)
# to an external file
if self.configParser.has_option("global", "hsplsFile"):
with open(self.configParser.get("global", "hsplsFile"),
"w") as f:
f.write(
etree.tostring(hsplSet,
pretty_print=True).decode())
if self.configParser.has_option("global", "msplsFile"):
with open(self.configParser.get("global", "msplsFile"),
"w") as f:
f.write(
etree.tostring(msplSet,
pretty_print=True).decode())
# Finally, sends everything to RabbitMQ.
self.send(hsplSet, msplSet)
elif len(fields) > 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1], IGNORECASE):
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.debug(
"Attack event (id: %d, severity: %s, type: %s, body: %s)" %
(identifier, severity, attackType, line))
key = "%d-%s-%s" % (identifier, severity, attackType)
if key not in self.attacks:
LOG.warning("Attack event without initial start message")
else:
self.attacks[key].addEvent("\t".join(fields[3:]))
else:
LOG.warning("Unknown message format: " + line)
channel.basic_ack(delivery_tag=method.delivery_tag)