def background(self, fullstr, sig): wrapper = SQLWrapper(conf_path, log) wrapper.connect() parser = signature_parser(wrapper, log, monitor_tab) #parser.wrapper.StartInsert() parser.sig_to_db(fullstr, sig, learning=learning_mode) parser.wrapper.StopInsert()
def __init__(self, pages_hit, rules_hit, rules_file, conf_file, log): self.log = log self.wrapper = SQLWrapper(conf_file, self.log) self.wrapper.connect() self.wrapper.setRowToDict() self.rules_hit = self.page_hit = 10 self.rules_list = [] self.final_rules = [] self.base_rules = [] self.pages_hit = pages_hit self.rules_hit = rules_hit self.core_msg = {} self.extract_core(rules_file)
def fill_db(files, conf_path): wrapper = SQLWrapper(conf_path, log) wrapper.connect() sig = '' count = 0 if re.match("[a-z0-9]+$", wrapper.dbname) == False: log.critial("Invalid dbname : " + wrapper.dbname) sys.exit(-1) wrapper.drop_database() wrapper.create_db() wrapper.select_db(wrapper.dbname) #wrapper.exec() log.critical("Filling db with %s (TABLES WILL BE DROPPED !)" % ' '.join(files)) parser = signature_parser(wrapper, log, None) parser.wrapper.StartInsert() for filename in files: with open(filename, 'r') as fd: for line in fd: fullstr = '' if 'NAXSI_FMT' in line: l = line.split(", ") date = ' '.join(l[0].split()[:2]) sig = l[0].split('NAXSI_FMT:')[1][1:] l = l[1:] request_args = {} for i in l: s = i.split(':') request_args[s[0]] = urllib.unquote(''.join(s[1:])) fullstr = request_args.get( 'request', 'None')[2:-1] + ' Referer : ' + request_args.get( 'referrer', ' "None"')[2:-1].strip( '"\n') + ',Cookie : ' + request_args.get( 'cookie', ' "None"')[2:-1] if sig != '' and fullstr != '': parser.sig_to_db(fullstr, sig, date=date, learning=learning_mode) count += 1 print(str(count) + " exceptions stored into database.") log.warning(str(count) + " exceptions stored into database.") parser.wrapper.StopInsert()
def fill_db(files, conf_path): wrapper = SQLWrapper(conf_path, log) wrapper.connect() sig = '' count = 0 if re.match("[a-z0-9]+$", wrapper.dbname) == False: log.critial("Invalid dbname : "+wrapper.dbname) sys.exit(-1) wrapper.drop_database() wrapper.create_db() wrapper.select_db(wrapper.dbname) #wrapper.exec() log.critical("Filling db with %s (TABLES WILL BE DROPPED !)" % ' '.join(files)) parser = signature_parser(wrapper, log, None) parser.wrapper.StartInsert() for filename in files: with open(filename, 'r') as fd: for line in fd: fullstr = '' if 'NAXSI_FMT' in line: l = line.split(", ") date = ' '.join(l[0].split()[:2]) sig = l[0].split('NAXSI_FMT:')[1][1:] l = l[1:] request_args = {} for i in l: s = i.split(':') request_args[s[0]] = urllib.unquote(''.join(s[1:])) fullstr = request_args.get('request', 'None')[2:-1] + ' Referer : ' + request_args.get('referrer', ' "None"')[2:-1].strip('"\n') + ',Cookie : ' + request_args.get('cookie', ' "None"')[2:-1] if sig != '' and fullstr != '': parser.sig_to_db(fullstr, sig, date=date, learning=learning_mode) count += 1 print(str(count)+" exceptions stored into database.") log.warning(str(count)+" exceptions stored into database.") parser.wrapper.StopInsert()
class rules_extractor: def __init__(self, pages_hit, rules_hit, rules_file, conf_file, log): self.log = log self.wrapper = SQLWrapper(conf_file, self.log) self.wrapper.connect() self.wrapper.setRowToDict() self.rules_hit = self.page_hit = 10 self.rules_list = [] self.final_rules = [] self.base_rules = [] self.pages_hit = pages_hit self.rules_hit = rules_hit self.core_msg = {} self.extract_core(rules_file) # self.log.warning( "Rules hit setting : "+str(self.rules_hit)) def extract_core(self, rules_file): try: fd = open(rules_file, 'r') for i in fd: if i.startswith('MainRule') or i.startswith('#@MainRule'): pos = i.find('id:') pos_msg = i.find('msg:') self.core_msg[i[pos + 3:i[pos + 3].find(';') - 1]] = i[pos_msg + 4:][:i[pos_msg + 4:].find('"')] fd.close() except: self.log.warning("Unable to open rules file.") pass def gen_basic_rules(self, url=None, srcip=None, dsthost=None, rule_id=None, exception_md5=None, exception_id=None): tmp_rules = [] self.base_rules = self.rules_list[:] def transform_to_dict(self, l): d = {} for i in l: if not d.has_key(i[0]): d[i[0]] = [] d[i[0]].append(i[1]) #elimininate duplicate ids in each value for i in d: d[i] = list(set(d[i])) return d def opti_rules_back(self): # rules of requests extracting optimized whitelists, from # more restrictive to less restrictive. opti_select_DESC = [ # select on url+var_name+zone+rule_id ("select count(*) as ct, e.rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, e.var_name," "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)" ), # select on var_name+zone+rule_id (unpredictable URL) ("select count(*) as ct, e.rule_id, e.zone, e.var_name, '' as url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id = u.url_id and c.id_exception = " "e.exception_id GROUP BY e.var_name, e.zone, e.rule_id HAVING (ct) > " "((select count(*) from connections)/1000)"), # select on zone+url+rule_id (unpredictable arg_name) ("select count(*) as ct, e.rule_id, e.zone, '' as var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, " "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)" ), # select on zone+url+var_name (unpredictable id) ("select count(*) as ct, 0 as rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, " "e.zone, e.var_name HAVING (ct) > ((select count(*) from connections)/1000)" ) ] for req in opti_select_DESC: # print "#------------------- first set of results" self.wrapper.execute(req) res = self.wrapper.getResults() for r in res: # print(r) if len(r['var_name']) > 0: self.try_append({ 'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'], 'var_name': r['var_name'], 'hcount': r['ct'], 'htotal': r['tot'], 'pcount': r['peer_count'], 'ptotal': r['ptot'], 'pratio': round((r['peer_count'] / float(r['ptot'])) * 100, 2), 'hratio': round((r['ct'] / float(r['tot'])) * 100, 2) }) else: self.try_append({ 'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'], 'var_name': '', 'hcount': r['ct'], 'htotal': r['tot'], 'ptotal': r['ptot'], 'pratio': round((r['peer_count'] / float(r['ptot'])) * 100, 2), 'hratio': round((r['ct'] / float(r['tot'])) * 100, 2), 'pcount': r['peer_count'] }) return self.base_rules, self.final_rules #returns true if whitelist 'target' is already handled by final_rules #does a dummy comparison and compares the counters def try_append(self, target, delmatch=False): count = 0 nb_rule = 0 uurl = set() # print "########## TRY APPEND :" # pprint.pprint(target) # print "--- vs ---" for z in self.final_rules[:]: if len(target['url']) > 0 and len( z['url']) > 0 and target['url'] != z['url']: continue if target['rule_id'] != 0 and z['rule_id'] != 0 and target[ 'rule_id'] != z['rule_id']: continue if len(target['zone']) > 0 and len( z['zone']) > 0 and target['zone'] != z['zone']: continue if len(target['var_name']) > 0 and len( z['var_name']) > 0 and target['var_name'] != z['var_name']: continue # pprint.pprint(z) #print "url:"+target['url'] uurl.add(z['url']) if delmatch is True: #print(z) self.final_rules.remove(z) else: nb_rule += 1 count += int(z['hcount']) if delmatch is True: return # No rules are matching this one, append. if not count and not nb_rule: self.final_rules.append(target) # print "Number of unique URLS covered "+str(len(uurl)) if target['hcount'] >= count and len(uurl) > self.pages_hit: self.try_append(target, True) self.final_rules.append(target) return if (target['hcount'] > count + 1) or (target['hcount'] >= count and nb_rule > self.rules_hit): self.try_append(target, True) self.final_rules.append(target) return def generate_stats(self): stats = "" self.wrapper.execute( "select count(distinct exception_id) as uniq_exception from exceptions" ) uniq_ex = self.wrapper.getResults()[0]['uniq_exception'] self.wrapper.execute( "select count(distinct peer_ip) as uniq_peer from connections") uniq_peer = self.wrapper.getResults()[0]['uniq_peer'] return "There is currently %s unique exceptions, with %s different peers that triggered rules." % ( uniq_ex, uniq_peer) def format_rules_output(self, opti_rules): r = '########### Optimized Rules Suggestion ##################\n' if not len(opti_rules): r += "#No rules to be generated\n" return opti_rules.sort(key=lambda k: (k['hratio'], k['pratio'])) _i = len(opti_rules) - 1 while _i >= 0: i = opti_rules[_i] _i = _i - 1 r += ("# total_count:" + str(i['hcount']) + " (" + str(i['hratio']) + "%), peer_count:" + str(i['pcount']) + " (" + str(i['pratio']) + "%)") r += " | " + self.core_msg.get(str(i['rule_id']), "?") + "\n" if (i['hratio'] < 5 and i['pratio'] < 5) or (i['pratio'] < 5): r += '#' r += 'BasicRule wl:' + str(i['rule_id']) + ' "mz:' if i['url'] is not None and len(i['url']) > 0: r += '$URL:' + i['url'] if i['rule_id'] == 1 and i['zone'] == "REQUEST": r += '";\n' continue if i['zone'] is not None and len(i['zone']) > 0: if i['url']: r += '|' if "FILE_EXT" in i['zone'] and i[ 'var_name'] is not None and len(i['var_name']) > 0: i['zone'] = i['zone'].replace("FILE_EXT", "BODY") if i['var_name'] is None: i['var_name'] = '' i['var_name'] = i['var_name'] + "|FILE_EXT" #(("|FILE_EXT" in i['zone'] and i['var_name'] is not None and len(i['var_name']) > 0) or if "|NAME" in i['zone'] and i['var_name'] is not None and len( i['var_name']) > 0: i['zone'] = i['zone'].replace("|NAME", "") if i['var_name'] is None: i['var_name'] = '' i['var_name'] = i['var_name'] + "|NAME" r += i['zone'] if i['var_name'] is not None and len(i['var_name']) > 0: # oooh, that must be bad. print "i:" pprint.pprint(i) r = r[:-len(i['zone'])] + "$" + r[-len(i['zone']):] r += "_VAR:" + i['var_name'] r += '";\n' return r
class rules_extractor: def __init__(self, pages_hit, rules_hit, rules_file, conf_file, log): self.log = log self.wrapper = SQLWrapper(conf_file, self.log) self.wrapper.connect() self.wrapper.setRowToDict() self.rules_hit = self.page_hit = 10 self.rules_list = [] self.final_rules = [] self.base_rules = [] self.pages_hit = pages_hit self.rules_hit = rules_hit self.core_msg = {} self.extract_core(rules_file) # self.log.warning( "Rules hit setting : "+str(self.rules_hit)) def extract_core(self, rules_file): try: fd = open(rules_file, 'r') for i in fd: if i.startswith('MainRule') or i.startswith('#@MainRule'): pos = i.find('id:') pos_msg = i.find('msg:') self.core_msg[i[pos + 3:i[pos + 3].find(';') - 1]] = i[pos_msg + 4:][:i[pos_msg + 4:].find('"')] fd.close() except: self.log.warning ("Unable to open rules file.") pass def gen_basic_rules(self,url=None, srcip=None, dsthost=None, rule_id=None, exception_md5=None, exception_id=None): tmp_rules = [] self.base_rules = self.rules_list[:] def transform_to_dict(self, l): d = {} for i in l: if not d.has_key(i[0]): d[i[0]] = [] d[i[0]].append(i[1]) #elimininate duplicate ids in each value for i in d: d[i] = list(set(d[i])) return d def opti_rules_back(self): # rules of requests extracting optimized whitelists, from # more restrictive to less restrictive. opti_select_DESC = [ # select on url+var_name+zone+rule_id ("select count(*) as ct, e.rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, e.var_name," "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)"), # select on var_name+zone+rule_id (unpredictable URL) ("select count(*) as ct, e.rule_id, e.zone, e.var_name, '' as url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id = u.url_id and c.id_exception = " "e.exception_id GROUP BY e.var_name, e.zone, e.rule_id HAVING (ct) > " "((select count(*) from connections)/1000)"), # select on zone+url+rule_id (unpredictable arg_name) ("select count(*) as ct, e.rule_id, e.zone, '' as var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, " "e.zone, e.rule_id HAVING (ct) > ((select count(*) from connections)/1000)"), # select on zone+url+var_name (unpredictable id) ("select count(*) as ct, 0 as rule_id, e.zone, e.var_name, u.url, count(distinct c.peer_ip) as peer_count, " "(select count(distinct peer_ip) from connections) as ptot, " "(select count(*) from connections) as tot " "from exceptions as e, urls as u, connections as c where c.url_id " "= u.url_id and c.id_exception = e.exception_id GROUP BY u.url, " "e.zone, e.var_name HAVING (ct) > ((select count(*) from connections)/1000)") ] for req in opti_select_DESC: # print "#------------------- first set of results" self.wrapper.execute(req) res = self.wrapper.getResults() for r in res: # print(r) if len(r['var_name']) > 0: self.try_append({'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'], 'var_name': r['var_name'], 'hcount': r['ct'], 'htotal': r['tot'], 'pcount':r['peer_count'], 'ptotal':r['ptot'], 'pratio': round((r['peer_count'] / float(r['ptot'])) * 100,2), 'hratio': round((r['ct'] / float(r['tot'])) * 100,2) }) else: self.try_append({'url': r['url'], 'rule_id': r['rule_id'], 'zone': r['zone'], 'var_name': '', 'hcount': r['ct'], 'htotal': r['tot'], 'ptotal':r['ptot'], 'pratio': round((r['peer_count'] / float(r['ptot'])) * 100,2), 'hratio': round((r['ct'] / float(r['tot'])) * 100,2), 'pcount':r['peer_count']}) return self.base_rules, self.final_rules #returns true if whitelist 'target' is already handled by final_rules #does a dummy comparison and compares the counters def try_append(self, target, delmatch=False): count=0 nb_rule=0 uurl = set() # print "########## TRY APPEND :" # pprint.pprint(target) # print "--- vs ---" for z in self.final_rules[:]: if len(target['url']) > 0 and len(z['url']) > 0 and target['url'] != z['url']: continue if target['rule_id'] != 0 and z['rule_id'] != 0 and target['rule_id'] != z['rule_id']: continue if len(target['zone']) > 0 and len(z['zone']) > 0 and target['zone'] != z['zone']: continue if len(target['var_name']) > 0 and len(z['var_name']) > 0 and target['var_name'] != z['var_name']: continue # pprint.pprint(z) #print "url:"+target['url'] uurl.add(z['url']) if delmatch is True: #print(z) self.final_rules.remove(z) else: nb_rule += 1 count += int(z['hcount']) if delmatch is True: return # No rules are matching this one, append. if not count and not nb_rule: self.final_rules.append(target) # print "Number of unique URLS covered "+str(len(uurl)) if target['hcount'] >= count and len(uurl) > self.pages_hit: self.try_append(target, True) self.final_rules.append(target) return if (target['hcount'] > count+1) or (target['hcount'] >= count and nb_rule > self.rules_hit): self.try_append(target, True) self.final_rules.append(target) return def generate_stats(self): stats = "" self.wrapper.execute("select count(distinct exception_id) as uniq_exception from exceptions") uniq_ex = self.wrapper.getResults()[0]['uniq_exception'] self.wrapper.execute("select count(distinct peer_ip) as uniq_peer from connections") uniq_peer = self.wrapper.getResults()[0]['uniq_peer'] return "There is currently %s unique exceptions, with %s different peers that triggered rules." % (uniq_ex, uniq_peer) def format_rules_output(self, opti_rules): r = '########### Optimized Rules Suggestion ##################\n' if not len(opti_rules): r+= "#No rules to be generated\n" return opti_rules.sort(key=lambda k: (k['hratio'], k['pratio'])) _i = len(opti_rules)-1 while _i >= 0: i = opti_rules[_i] _i = _i - 1 r += ("# total_count:"+str(i['hcount'])+" ("+str(i['hratio'])+ "%), peer_count:"+str(i['pcount'])+" ("+str(i['pratio'])+"%)") r += " | "+self.core_msg.get(str(i['rule_id']), "?")+"\n" if (i['hratio'] < 5 and i['pratio'] < 5) or (i['pratio'] < 5): r += '#' r += 'BasicRule wl:' + str(i['rule_id']) + ' "mz:' if i['url'] is not None and len(i['url']) > 0: r += '$URL:' + i['url'] if i['rule_id'] == 1 and i['zone'] == "REQUEST": r += '";\n' continue if i['zone'] is not None and len(i['zone']) > 0: if i['url']: r += '|' if "FILE_EXT" in i['zone'] and i['var_name'] is not None and len(i['var_name']) > 0: i['zone'] = i['zone'].replace("FILE_EXT", "BODY") if i['var_name'] is None: i['var_name'] = '' i['var_name'] = i['var_name']+"|FILE_EXT" #(("|FILE_EXT" in i['zone'] and i['var_name'] is not None and len(i['var_name']) > 0) or if "|NAME" in i['zone'] and i['var_name'] is not None and len(i['var_name']) > 0: i['zone'] = i['zone'].replace("|NAME", "") if i['var_name'] is None: i['var_name'] = '' i['var_name'] = i['var_name']+"|NAME" r += i['zone'] if i['var_name'] is not None and len(i['var_name']) > 0: # oooh, that must be bad. print "i:" pprint.pprint(i) r = r[:-len(i['zone'])]+"$"+r[-len(i['zone']):] r += "_VAR:"+i['var_name'] r += '";\n' return r