def process_IN_CLOSE_WRITE(self, event):
"""
Handles a file creation.
@param event: The file event.
"""
LOG.debug("Callback from event in directory")
try:
# First, translate the CSV in HSPL, MSPL sets
[hsplSet, msplSet] = self.getMSPLsFromFile(
event.pathname, self.configParser.get("global",
"landscapeFile"))
# Then, if extra logging is activated, print HSPL (and/or MSPL)
# to an external file
if self.configParser.has_option("global", "hsplsFile"):
with open(self.configParser.get("global", "hsplsFile"),
"w") as f:
f.write(
etree.tostring(hsplSet, pretty_print=True).decode())
if self.configParser.has_option("global", "msplsFile"):
with open(self.configParser.get("global", "msplsFile"),
"w") as f:
f.write(
etree.tostring(msplSet, pretty_print=True).decode())
# Finally, sends everything to RabbitMQ.
self.send(hsplSet, msplSet)
except BaseException as e:
LOG.critical(str(e))
if self.channel is not None:
if not self.channel.is_closed:
self.channel.close()
def on_queue_declareok(self, frame):
LOG.debug("Queue declare is ok, binding queue")
queue = self.configParser.get("global", "serverQueue")
exchange = self.configParser.get("global", "serverExchange")
topic = self.configParser.get("global", "serverTopic")
self.r_channel.queue_bind(self.on_bindok, queue, exchange, topic)
def on_channel_open(self, channel):
LOG.debug("Channel open, declaring exchange")
exchange = self.configParser.get("global", "serverExchange")
self.r_channel = channel
self.r_channel.exchange_declare(self.on_exchange_declareok, exchange,
"topic")
def on_connection_closed(self, connection, reply_code, reply_text):
LOG.debug("Detected a closed connection... Reconnecting in a while...")
self.r_channel = None
if not self.r_closingConnection:
self.r_connection.add_timeout(5, self.reconnect)
else:
self.r_connection.ioloop.stop()
def getMSPLsFromList(self, identifier, severity, attackType, attackList,
landscapeFileName, anomaly_name):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param identifier: the attack id.
@param severity: the attack severity.
@param attackType: the attack type.
@param attackList: the list to parse.
@param landscapeFileName: the name of the landscape file to parse.
@return: The HSPL set and MSPL set that can mitigate the attack. It is
None if the attack is not manageable.
@raise SyntaxError: When the generated XML is not valid.
"""
attack = self.parser.getAttackFromList(identifier, severity,
attackType, attackList,
anomaly_name)
LOG.debug("Got attack from list")
landscape = self.parser.getLandscape(landscapeFileName)
LOG.debug("Got landscape")
recipes = self.recipesReasoner.getRecipes(attack, landscape)
LOG.debug("Got recipes")
hsplSet = self.hsplReasoner.getHSPLs(attack, recipes, landscape)
LOG.debug("Got HSPL set")
msplSet = self.msplReasoner.getMSPLs(hsplSet, landscape, anomaly_name)
LOG.debug("Got MSPL set")
if hsplSet is None or msplSet is None:
return None
else:
return [hsplSet, msplSet]
def connect(self):
LOG.debug("RabbitMQ connect invoked")
address = self.configParser.get("global", "serverAddress")
port = self.configParser.getint("global", "serverPort")
return pika.SelectConnection(pika.ConnectionParameters(host=address,
port=port),
self.on_connection_open,
self.on_connection_error,
stop_ioloop_on_close=False)
def send(self, hsplSet, msplSet):
"""
Sends the policies to the dashboard.
@param hsplSet: the HSPL set.
@param msplSet: the MSPL set.
"""
if (self.configParser.has_option("global", "dashboardHost")
and self.configParser.has_option("global", "dashboardPort")
and self.configParser.has_option("global", "dashboardExchange")
and self.configParser.has_option("global", "dashboardTopic")
and self.configParser.has_option("global", "dashboardAttempts")
and self.configParser.has_option("global",
"dashboardRetryDelay")):
host = self.configParser.get("global", "dashboardHost")
port = self.configParser.getint("global", "dashboardPort")
connectionAttempts = self.configParser.getint(
"global", "dashboardAttempts")
retryDelay = self.configParser.getint("global",
"dashboardRetryDelay")
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=host,
port=port,
connection_attempts=connectionAttempts,
retry_delay=retryDelay,
blocked_connection_timeout=300))
self.channel = connection.channel()
self.channel.exchange_declare(exchange=self.configParser.get(
"global", "dashboardExchange"),
exchange_type="topic")
LOG.info("Connected to the dashboard at " + host + ":" + str(port))
hsplString = etree.tostring(hsplSet).decode()
msplString = etree.tostring(msplSet).decode()
content = self.configParser.get("global", "dashboardContent")
if content == "HSPL":
message = hsplString
elif content == "MSPL":
message = msplString
else:
message = hsplString + msplString
LOG.info("Pushing the remediation to the dashboard")
exchange = self.configParser.get("global", "dashboardExchange")
topic = self.configParser.get("global", "dashboardTopic")
self.channel.basic_publish(exchange=exchange,
routing_key=topic,
body=message)
LOG.debug("Dashboard RabbitMQ exchange: " + exchange + " topic: " +
topic)
LOG.info("Remediation forwarded to the dashboard")
self.channel.close()
LOG.info("Connection with the dashboard closed")
def listenFolder(self):
"""
Starts the CyberTop policy engine by listening to a folder.
"""
LOG.debug("Request for directory listening")
directory = self.configParser.get("global", "watchedDirectory")
LOG.debug("Starting directory listener: " + directory)
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm, self)
wm.add_watch(directory,
pyinotify.IN_CLOSE_WRITE,
rec=True,
auto_add=True)
notifier.loop(daemonize=False)
def on_bindok(self, frame):
LOG.debug("Binding queue is ok, start consuming")
queue = self.configParser.get("global", "serverQueue")
self.r_channel.add_on_cancel_callback(self.on_consumer_cancelled)
self.r_channel.basic_consume(self.processMessage, queue=queue)
def on_exchange_declareok(self, unused_frame):
LOG.debug("Exchange declare is ok, declaring queue")
queue = self.configParser.get("global", "serverQueue")
self.r_channel.queue_declare(self.on_queue_declareok,
queue,
durable=True)
def open_channel(self):
LOG.debug("Opening channel")
self.r_connection.channel(on_open_callback=self.on_channel_open)
def getHSPLs(self, attack, recipes, landscape):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param attack: The attack to mitigate.
@param recipes: The recipes to use.
@param landscape: The landscape.
@return: The XML HSPL set that can mitigate the attack. It is None if no recipe is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if recipes is None:
return None
schema = etree.XMLSchema(etree.parse(getHSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
for recipe in recipes:
hsplSet = etree.SubElement(recommendations,
"{%s}hspl-set" % getHSPLNamespace(),
nsmap={
None: getHSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
recipeName = recipe.findtext("{%s}name" % getRecipeNamespace())
recipeAction = recipe.findtext("{%s}action" % getRecipeNamespace())
recipeSubjectAnyAddress = recipe.findtext(
"{%s}subject-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeSubjectAnyPort = recipe.findtext(
"{%s}subject-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyAddress = recipe.findtext(
"{%s}object-constraints/{%s}any-address" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeObjectAnyPort = recipe.findtext(
"{%s}object-constraints/{%s}any-port" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeType = recipe.findtext(
"{%s}traffic-constraints/{%s}type" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeMaxConnections = recipe.findtext(
"{%s}traffic-constraints/{%s}max-connections" %
(getRecipeNamespace(), getRecipeNamespace()))
recipeRateLimit = recipe.findtext(
"{%s}traffic-constraints/{%s}rate-limit" %
(getRecipeNamespace(), getRecipeNamespace()))
# Adds the context.
context = etree.SubElement(hsplSet,
"{%s}context" % getHSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getHSPLNamespace()).text = str(attack.severity)
etree.SubElement(context, "{%s}type" %
getHSPLNamespace()).text = attack.type
etree.SubElement(
context, "{%s}timestamp" %
getHSPLNamespace()).text = attack.getTimestamp().isoformat()
# Filters the events.
events = []
recipeFilters = recipe.find("{%s}filters" % getRecipeNamespace())
evaluation = "or"
if recipeFilters is None:
events = attack.events
else:
if "evaluation" in recipeFilters.attrib.keys():
evaluation = recipeFilters.attrib["evaluation"]
for i in attack.events:
if evaluation == "or":
test = False
else:
test = True
for j in self.pluginManager.getPluginsOfCategory("Filter"):
pluginTag = j.details.get("Core", "Tag")
filterValues = recipeFilters.findall(
"{%s}%s" % (getRecipeNamespace(), pluginTag))
for k in filterValues:
t = j.plugin_object.filter(k.text, i)
if evaluation == "or":
test = test or t
else:
test = test and t
if not test:
events.append(i)
# Adds an HSPL for each event.
count = 0
for i in events:
count += 1
hspl = etree.SubElement(hsplSet,
"{%s}hspl" % getHSPLNamespace())
etree.SubElement(
hspl, "{%s}name" %
getHSPLNamespace()).text = "%s #%d" % (recipeName, count)
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.target)
targetAddress = m.group(1)
targetPort = m.group(4)
if recipeSubjectAnyAddress is not None:
targetAddress = "*"
if recipeSubjectAnyPort is not None:
targetPort = "*"
etree.SubElement(
hspl, "{%s}subject" %
getHSPLNamespace()).text = "%s:%s" % (targetAddress,
targetPort)
etree.SubElement(hspl, "{%s}action" %
getHSPLNamespace()).text = recipeAction
m = re.match("(\d+\.\d+\.\d+\.\d+(/\d+)?)(:(\d+|\*|any))?",
i.attacker)
attackerAddress = m.group(1)
attackerPort = m.group(4)
if recipeObjectAnyAddress is not None:
attackerAddress = "*"
if recipeObjectAnyPort is not None:
attackerPort = "*"
etree.SubElement(
hspl, "{%s}object" %
getHSPLNamespace()).text = "%s:%s" % (attackerAddress,
attackerPort)
trafficConstraints = etree.SubElement(
hspl, "{%s}traffic-constraints" % getHSPLNamespace())
if recipeType is not None:
eventType = recipeType
else:
eventType = i.fields["protocol"]
etree.SubElement(trafficConstraints, "{%s}type" %
getHSPLNamespace()).text = eventType
if eventType == "TCP" and recipeMaxConnections is not None:
etree.SubElement(
trafficConstraints, "{%s}max-connections" %
getHSPLNamespace()).text = recipeMaxConnections
if recipeRateLimit is not None:
etree.SubElement(trafficConstraints, "{%s}rate-limit" %
getHSPLNamespace()).text = recipeRateLimit
LOG.debug(etree.tostring(recommendations, pretty_print=True).decode())
if schema.validate(recommendations):
return self.__cleanAndMerge(recommendations)
else:
LOG.critical("Invalid HSPL recommendations generated.")
raise SyntaxError("Invalid HSPL recommendations generated.")
def __init__(self,
configurationFileName=None,
logConfigurationFileName=None):
"""
Constructor.
@param configurationFileName: the name of the configuration file to
parse.
@param logConfigurationFileName: the name of the log configuration file
to use.
"""
# Configures the logging.
log.load_settings(logConfigurationFileName)
# Configures the configuration file parser.
self.configParser = ConfigParser()
if configurationFileName is None:
c = self.configParser.read(getConfigurationFile())
else:
c = self.configParser.read(configurationFileName)
if len(c) > 0:
LOG.debug("Configuration file '%s' read." % c[0])
else:
LOG.critical("Cannot read the configuration file from '%s'." %
configurationFileName)
raise IOError("Cannot read the configuration file from '%s'" %
configurationFileName)
# Configures the plug-ins.
self.pluginManager = PluginManager()
self.pluginManager.setPluginPlaces([getPluginDirectory()])
self.pluginManager.setCategoriesFilter({
"Action": ActionPlugin,
"Parser": ParserPlugin,
"Filter": FilterPlugin
})
self.pluginManager.collectPlugins()
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Parser"))
if pluginsCount > 1:
LOG.info("Found %d attack event parser plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event parser plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Filter"))
if pluginsCount > 1:
LOG.info("Found %d attack event filter plug-ins.", pluginsCount)
else:
LOG.info("Found %d attack event filter plug-in.", pluginsCount)
pluginsCount = len(self.pluginManager.getPluginsOfCategory("Action"))
if pluginsCount > 1:
LOG.info("Found %d action plug-ins.", pluginsCount)
else:
LOG.info("Found %d action plug-in.", pluginsCount)
# Loads all the sub-modules.
self.parser = Parser(self.configParser, self.pluginManager)
self.recipesReasoner = RecipesReasoner(self.configParser,
self.pluginManager)
self.hsplReasoner = HSPLReasoner(self.configParser, self.pluginManager)
self.msplReasoner = MSPLReasoner(self.configParser, self.pluginManager)
# Starts with no attack info.
self.attacks = {}
# Connection to the DARE rabbitMQ queue
self.r_connection = None
self.r_channel = None
self.r_closingConnection = False
LOG.info("CyberSecurity Topologies initialized.")
def on_connection_open(self, new_connection):
LOG.debug('Opened connection')
self.r_connection.add_on_close_callback(self.on_connection_closed)
self.open_channel()
def processMessage(self, channel, method, header, body):
"""
Handles a RabbitMQ message.
@param channel: The channel.
@param method: The method.
@param header: The message header.
@param body: The message body.
"""
LOG.debug("Callback from event in RabbitMQ")
line = body.decode()
dialect = Sniffer().sniff(line)
fields = []
for i in reader([line], dialect):
fields += i
LOG.debug("DARE RabbitMQ message: " + line)
if len(fields) == 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1],
IGNORECASE) and fields[3] == "start":
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.info("Attack started (id: %d, severity: %s, type: %s)" %
(identifier, severity, attackType))
key = "%d-%s-%s" % (identifier, severity, attackType)
if key in self.attacks:
LOG.warning("Duplicate start message")
else:
self.attacks[key] = AttackInfo(identifier, severity,
attackType)
elif len(fields) == 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1],
IGNORECASE) and fields[3] == "stop":
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.info("Attack stopped (id: %d, severity: %s, type: %s)" %
(identifier, severity, attackType))
# store the anomaly detection name in a variable
anomaly_name = attackType
LOG.debug("Anomaly name is: " + anomaly_name)
key = "%d-%s-%s" % (identifier, severity, attackType)
if key not in self.attacks:
LOG.warning("Stop message without initial start message")
else:
attackInfo = self.attacks[key]
self.attacks.pop(key)
identifier = attackInfo.getIdentifier()
severity = attackInfo.getSeverity()
attackType = attackInfo.getType()
events = attackInfo.getEvents()
landscapeFileName = self.configParser.get(
"global", "landscapeFile")
LOG.debug("Get mspls from list")
# First, translate the CSV in HSPL, MSPL sets
[hsplSet,
msplSet] = self.getMSPLsFromList(identifier, severity,
attackType, events,
landscapeFileName,
anomaly_name)
LOG.debug("Got mspls from list")
# Then, if extra logging is activated, print HSPL (and/or MSPL)
# to an external file
if self.configParser.has_option("global", "hsplsFile"):
with open(self.configParser.get("global", "hsplsFile"),
"w") as f:
f.write(
etree.tostring(hsplSet,
pretty_print=True).decode())
if self.configParser.has_option("global", "msplsFile"):
with open(self.configParser.get("global", "msplsFile"),
"w") as f:
f.write(
etree.tostring(msplSet,
pretty_print=True).decode())
# Finally, sends everything to RabbitMQ.
self.send(hsplSet, msplSet)
elif len(fields) > 4 and fields[0].isdigit() and match(
"(very\s+)?(low|high)", fields[1], IGNORECASE):
identifier = int(fields[0])
severity = " ".join(fields[1].lower().split())
attackType = fields[2]
LOG.debug(
"Attack event (id: %d, severity: %s, type: %s, body: %s)" %
(identifier, severity, attackType, line))
key = "%d-%s-%s" % (identifier, severity, attackType)
if key not in self.attacks:
LOG.warning("Attack event without initial start message")
else:
self.attacks[key].addEvent("\t".join(fields[3:]))
else:
LOG.warning("Unknown message format: " + line)
channel.basic_ack(delivery_tag=method.delivery_tag)
def on_consumer_cancelled(self, frame):
LOG.debug("Consumer cancelled")
if self.r_channel:
self.r_channel.close()
def on_connection_error(self, connection, error):
LOG.debug("Connection error: " + str(error))
time.sleep(5)
self.reconnect()
def reconnect(self):
self.r_connection.ioloop.stop()
LOG.debug("Reconnecting now")
if not self.r_closingConnection:
self.r_connection = self.connect()
self.r_connection.ioloop.start()
def getMSPLs(self, hsplRecommendations, landscape, anomaly_name):
"""
Retrieve the HSPLs that can be used to mitigate an attack.
@param recommendations: The HSPL recommendations to use.
@param landscape: The landscape.
@return: The XML MSPL set that can mitigate the attack. It is None if no HSPL is available.
@raise SyntaxError: When the generated XML is not valid.
"""
if hsplRecommendations is None:
return None
schema = etree.XMLSchema(etree.parse(getMSPLXSDFile()))
recommendations = etree.Element("{%s}recommendations" %
getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
for hsplSet in hsplRecommendations:
msplSet = etree.SubElement(recommendations,
"{%s}mspl-set" % getMSPLNamespace(),
nsmap={
None: getMSPLNamespace(),
"xsi": getXSINamespace()
})
# Gather some data about the recipe.
msplSeverity = hsplSet.findtext(
"{%s}context/{%s}severity" %
(getHSPLNamespace(), getHSPLNamespace()))
msplType = hsplSet.findtext(
"{%s}context/{%s}type" %
(getHSPLNamespace(), getHSPLNamespace()))
msplTimestamp = hsplSet.findtext(
"{%s}context/{%s}timestamp" %
(getHSPLNamespace(), getHSPLNamespace()))
# Adds the context.
context = etree.SubElement(msplSet,
"{%s}context" % getMSPLNamespace())
etree.SubElement(context, "{%s}severity" %
getMSPLNamespace()).text = msplSeverity
#etree.SubElement(context, "{%s}type" % getMSPLNamespace()).text = msplType
etree.SubElement(context, "{%s}type" %
getMSPLNamespace()).text = anomaly_name
etree.SubElement(context, "{%s}timestamp" %
getMSPLNamespace()).text = msplTimestamp
# Finds a plug-in that can create a configured IT resource.
[plugin, identifier] = self.__findLocation(hsplSet, landscape)
plugin.plugin_object.setup(self.configParser)
LOG.info("Check if VNSFO API call (experimental) is enabled")
# Check if VNSFO is integrated to recommendations engine
vnsfo_integration = self.configParser.getboolean(
"vnsfo", "enable_vnsfo_api_call")
if vnsfo_integration:
# Creates the IT resource.
LOG.info("Experimental: contact vNSFO API")
vnsfo_base_url = self.configParser.get("vnsfo",
"vnsfo_base_url")
vnsfo_timeout = int(
self.configParser.get("vnsfo", "vnsfo_timeout"))
if not vnsfo_base_url:
LOG.info("VNSFO base URL empty. Fallback to stable.")
else:
LOG.info("Retrieving VNSF running ID for: " + identifier)
vnfr_id = retrieve_vnsfr_id(vnsfo_base_url, identifier,
anomaly_name, vnsfo_timeout)
if vnfr_id:
LOG.info("VNSF running ID is: " + vnfr_id)
identifier = vnfr_id
else:
LOG.info("Stable solution selected.")
itResource = etree.SubElement(
msplSet, "{%s}it-resource" % getMSPLNamespace(),
{"id": identifier})
# Calls the plug-in to configure the IT resource.
plugin.plugin_object.configureITResource(itResource, hsplSet)
if schema.validate(recommendations):
LOG.debug(
etree.tostring(recommendations, pretty_print=True).decode())
return recommendations
else:
LOG.critical("Invalid MSPL recommendations generated.")
raise SyntaxError("Invalid MSPL recommendations generated.")
def __cleanAndMerge(self, recommendations):
"""
Polish an HSPL set by removing the duplicate HSPLs and merging them together, if needed. We only work on the objects.
@param recommendations: The HSPL recommendations set to use.
@return: The cleaned HSPL set.
"""
hsplMergeInclusions = int(
self.configParser.getboolean("global", "hsplMergeInclusions"))
hsplMergeWithAnyPorts = int(
self.configParser.getboolean("global", "hsplMergeWithAnyPorts"))
hsplMergeWithSubnets = int(
self.configParser.getboolean("global", "hsplMergeWithSubnets"))
if not hsplMergeInclusions and not hsplMergeWithAnyPorts and not hsplMergeWithSubnets:
return recommendations
count = 0
for hsplSet in recommendations:
# Pass 0: create the map.
hsplMap = HSPLMap()
for i in hsplSet:
if i.tag == "{%s}hspl" % getHSPLNamespace():
hsplMap.add(i)
# Pass 1: removes the included HSPLs.
if hsplMergeInclusions:
includedHSPLs = self.__mergeInclusions(hsplSet, hsplMap)
if includedHSPLs > 1:
LOG.debug("%d included HSPLs removed for the HSPL set %d.",
includedHSPLs, count)
else:
LOG.debug("%d included HSPL removed for the HSPL set %d.",
includedHSPLs, count)
# Pass 2: merges the IP address using * as the port number.
if hsplMergeWithAnyPorts:
mergedHSPLs = self.__mergeWithAnyPorts(hsplSet, hsplMap)
if mergedHSPLs > 1:
LOG.debug(
"%d HSPLs merged using any ports for the HSPL set %d.",
mergedHSPLs, count)
else:
LOG.debug(
"%d HSPL merged using any port for the HSPL set %d.",
mergedHSPLs, count)
# Pass 3: merges the HSPLs, if needed.
if hsplMergeWithSubnets:
mergedHSPLs = self.__mergeWithSubnets(hsplSet, hsplMap)
if mergedHSPLs > 1:
LOG.debug(
"%d HSPLs merged using subnets for the HSPL set %d.",
mergedHSPLs, count)
else:
LOG.debug(
"%d HSPL merged using subnets for the HSPL set %d.",
mergedHSPLs, count)
return recommendations