def recreate_bloom_filter():
get_IOCs()
iocs = ioc_csv('misp_events')
crypto = Crypto('bloom_filter', conf, metadata)
# Create a dico of rules
iocDic = readMisp_parsing(iocs, crypto)
# Save bloom filter
crypto.write_bloom()
def test_meta(self):
# Test feature two.
cry = Crypto('pbkdf2', self.conf, self.metadata)
ioc = OrderedDict()
ioc['ip-dst'] = "192.168.0.0"
ioc['url'] = "test.com"
rule = cry.create_rule(ioc, "Hello, this is the message!")
rule['salt'] = b64decode(rule['salt'])
rule['nonce'] = b64decode(rule['nonce'])
rule['attributes'] = rule['attributes'].split('||')
rule['ciphertext-check'] = b64decode(rule['ciphertext-check'])
rule['ciphertext'] = b64decode(rule['ciphertext'])
queue = SimpleQueue()
cry.match(ioc, rule, queue)
self.assertTrue(not queue.empty())
def __init__(self, conf, metadata=None, cryptoName=''):
self.conf = conf
self.Crypto = ChooseCrypto(cryptoName, conf, metadata)
# Set up bloom
self.bloom = BF(conf,
metadata=metadata,
rate=conf['bloomy']['fp_rate'])
# Make it faster without modifying
self.cacheVal = ""
self.cacheBool = True
def saveIOCs():
metaParser = configparser.ConfigParser()
try:
metaParser.read(conf['rules']['location'] + '/metadata')
metadata = metaParser._sections
except:
print('Rules must have already been created for adding news')
sys.exit(1)
os.remove(conf['rules']['location'] + '/metadata')
crypto = Crypto(conf["rules"]["cryptomodule"], conf, metadata)
ruleFiles = {}
for name in os.listdir(conf['rules']['location']):
ruleFiles[(name.split('.')[0]).split('_')[0]] = name
# Create a dico of rules
iocDic = readMisp_parsing(IOCs, crypto)
# For each type add to file if exist
iocNewType = {}
bar = ProgressBar()
for iocType in bar(iocDic.keys()):
# Exist rules of the same type
try:
filename = ruleFiles[iocType]
with open(conf['rules']['location'] + '/' + filename, 'r') as f:
header = f.readline()[:-1]
rowsNames = header.split('\t')
iocLines = create_ioc_lines(rowsNames, iocDic[iocType])
with open(conf['rules']['location'] + '/' + filename, 'a') as f:
f.write(iocLines)
except Exception:
iocNewType[iocType] = iocDic[iocType]
if len(iocNewType) > 0:
store_rules(iocNewType, conf)
# ReSave metaparameter important for bloom filters
printv("Rewrite metadata")
crypto.save_meta()
class Bloomy(Crypto):
def __init__(self, conf, metadata=None, cryptoName=''):
self.conf = conf
self.Crypto = ChooseCrypto(cryptoName, conf, metadata)
# Set up bloom
self.bloom = BF(conf,
metadata=metadata,
rate=conf['bloomy']['fp_rate'])
# Make it faster without modifying
self.cacheVal = ""
self.cacheBool = True
def create_rule(self, ioc, message):
self.bloom.create_rule(ioc, message)
return self.Crypto.create_rule(ioc, message)
def match(self, attributes, rule, queue):
if self.cacheVal != attributes:
self.cacheVal = attributes
if len(self.bloom.check(attributes, rule)) > 0:
self.cacheBool = True
self.Crypto.match(attributes, rule, queue)
else:
self.cacheBool = False
else:
if self.cacheBool == True:
self.Crypto.match(attributes, rule, queue)
def save_meta(self):
conf = self.conf
# Create bloom filter
self.bloom.write_bloom()
# Save metadata
self.Crypto.save_meta()
# Add bloomy_
metaParser = ConfigParser()
metaParser.read(conf['rules']['location'] + '/metadata')
metadata = metaParser._sections
cryptoName = 'bloomy_' + metadata['crypto']['name']
metaParser['crypto']['name'] = cryptoName
with open(conf['rules']['location'] + '/metadata', 'w') as config:
metaParser.write(config)
ip=['ip-dst=192.168.' + str(ip4_0) + '.' + str(ip4_1)]
argument_matching(crypto, ip)
########
# Main #
########
if __name__ == "__main__":
conf = Configuration()
rules = list()
# Get configuration
metaParser = configparser.ConfigParser()
metaParser.read(conf['rules']['location'] + "/metadata")
metadata = metaParser._sections
# Choose crypto
crypto = Crypto(metadata['crypto']['name'], conf, metadata)
if not os.path.exists(conf['rules']['location']):
sys.exit("No rules found.")
# Get all files attributes
filenames = os.listdir(conf['rules']['location'])
for name in filenames:
split = (name.split('.')[0]).split('_')
file_attributes[name] = split
if args.input == "redis":
redis_matching(crypto)
elif args.input == "rangeip":
rangeip_test(crypto)
shutil.rmtree(conf['rules']['location'])
os.mkdir(conf['rules']['location'])
# Fill IOC list
printv("Get IOCs from " + args.misp)
if args.misp == 'web':
ioc_web()
elif args.misp == 'mysql':
ioc_mysql()
elif args.misp == 'res':
ioc_csv()
else:
sys.exit(
'misp argument is miss configured. Please select web, res or mysql'
)
# Choose crypto system
crypto = Crypto(conf["rules"]["cryptomodule"], conf)
# Parse IOCs
iocDic = parsing(IOCs, crypto)
store_rules(iocDic)
# Create metadata (End function for Crypto modules)
printv("Create metadata")
crypto.save_meta()
else:
def printv(val):
pass