def register_source(self, **kwargs):
""" Register source data, and adds to database.
If source exsists, it will be updated. If the source does not excist it will be added.
"""
try: # Check if source is active, deactivate if necessary.
if 'active' not in kwargs or kwargs['active'] == 'off':
active = 0
else:
active = 1
url = kwargs['source-url']
except KeyError:
raise
if 'source-id' in kwargs:
id = kwargs['source-id']
else:
id = None
if id:
sourceUrl = Session.query(UpdateSource).get(id)
sourceUrl.url = url
sourceUrl.active = active
else:
sourceUrl = UpdateSource(active, url)
Session.merge(sourceUrl)
raise cherrypy.HTTPRedirect('/rules/')
def save_object(self, **kwargs):
""" Add the contents of a policy object to the database. """
print '\n\n Save object \n\n'
try:
content = escape_string(kwargs['object-content'])
object_id = int(kwargs['object-id'])
type = kwargs['object-type']
item_id = kwargs['policy-id']
if object_id == 0:
policy_item = Session.query(PolicyChain).filter(
PolicyChain.id == item_id).one()
policy_object = PolicyObject(type, content)
Session.add(policy_object)
Session.flush()
policy_item.policyobject_id = policy_object.id
else:
policy_object= Session.query(PolicyObject).filter(PolicyObject.id == object_id).one()
policy_object.contents = content
policy_object.type = type
Session.merge(policy_object)
Session.flush()
except KeyError as e:
print e
except NoResultFound as e:
#policy_object = PolicyObject(type, content)
#Session.add(policy_object)
#Session.flush()
print 'no result found'
pass
def add_rules_to_database(self, file_id, ruleList):
""" Add rules to database based on file """
stored_rules = Session.query(Rules).filter(Rules.file == file_id).all() # Returns entire set of rules
stored_list = {} # Cache of already accessed rules
rules_to_add = [] # List of rule to add to database
for new_rule in ruleList:
try:
try:
if int(new_rule.sid) not in stored_list or stored_list[new_rule.sid] is None:
stored_list[int(new_rule.sid)] = self._find_local_rule(stored_rules, int(new_rule.sid))
stored_local_rules = stored_list[int(new_rule.sid)]
if stored_local_rules is None or len(stored_local_rules) == 0:
# print 'print %s is not in DB. Type %s' % (int(new_rule.sid), type(new_rule.sid))
# for rules in stored_rules:
# print 'Sid: %s Rev: %s\nRule: %s' % (rules.sid, rules.rev, rules.rule)
raise NoResultFound
old_rule = stored_local_rules[0]
# print '--#--\n SID: %s' % (new_rule.sid)
# print '---\n ID: %s \n Rule: %s --#--' % (old_rule.sid, old_rule.rule)
# print '-#- \n %s \n -#- \n %s-#-' % (old_rule, new_rule)
if int(old_rule.rev) < int(new_rule.rev) or str(old_rule.rule) != str(new_rule.raw):
old_rule = self.deactivate_rule(old_rule)
print "Different revision or content %s" % old_rule.sid
raise NoResultFound
if new_rule.active != old_rule.active:
print "Rule is deactiviated: %s" % old_rule.sid
old_rule = deactiviate_rule(old_rule)
stored_local_rules[0] = old_rule
stored_list[int(new_rule.sid)] = stored_local_rules
except NoResultFound: # No result found, add rule
new_rule_add = Rules(new_rule, file_id)
rules_to_add.append(new_rule_add)
except IndexError as e:
print "List index: %s" % (e)
print stored_local_rules
exit(1)
except TypeError as e:
print "Type error: %s" % (e)
# print type(new_rule.sid)
# print stored_local_rules
exit(1)
except OperationalError as e:
print "EXCEPTION in add_rules_to_database\nError information: %s" % (e) # TODO: Needs to f**k off
Session.add_all(rules_to_add)
Session.flush()
def save_sensor_data(self, **kwargs):
""" Get a sensors data provided bu a form, and either change it or add new. """
try:
""" Retrieves information from web request. """
addName = escape_string(kwargs['sensor_name'])
addIp = escape_string(kwargs['sensor_ip'])
addLocation = kwargs['sensor_location']
addDescription = escape_string(kwargs['sensor_description'])
sensor_id = kwargs['sensor_id']
except KeyError as e:
print 'There was a key error %s' % (e)
try:
""" In case sensor does not excist. """
if sensor_id == 0:
raise NoResultFound
sensor = Session.query(Sensor).filter(Sensor.id == sensor_id).one()
sensor.name = addName
sensor.ip = addIp
sensor.description = addDescription
Session.merge(sensor)
except NoResultFound: # Sensor does not exsist, add new.
new_sensor = Sensor(addName, addIp, addLocation, addDescription)
Session.add(new_sensor)
finally:
Session.flush()
Session.flush()
def add_object(self, **kwargs):
""" Creates a new policy object. """
try:
if kwargs['object_id'] is None or not kwargs['object_id'].isdigit():
raise TypeError('Policy ID is either None or ID is not int')
if kwargs ['object_content'] is None:
raise TypeError('Policy content None')
except KeyError as e:
return 'Key %s does not exist' % e
except TypeError as e:
return '<p>An error occured</p><p><b>Errorinfo:</b><br/>%s' % e
policy_id = kwargs['object_id']
try:
policy_object = Session.query(PolicyObject).filter(PolicyObject.id == policy_id).one()
policy_object.contents = escape_string(kwargs['object_content'])
Session.merge(policy_object)
Session.flush()
except NoResultFound:
policy_object = PolicyObject(contents = escape_string(kwargs['object_content']))
Session.add(policy_object)
Session.flush()
finally:
raise cherrypy.HTTPRedirect("/policy/")
def register_update_in_database(self, urlid=0):
""" Register download in database """
if urlid != 0:
try:
UpdateItem = UpdateLog(urlid, datetime.now())
Session.add(UpdateItem)
return UpdateItem.id
except:
print ("Error while registering update with database")
raise
return 0
def delete_sensor(self, **kwargs):
""" Deactivates a sensor in the system. """
try:
id = kwargs['sensor_id']
sensor = Session.query(Sensor).filter(Sensor.id == id).one()
sensor.active = 0
Session.merge(sensor)
Session.flush()
except NoResultFound:
print 'No result found'
except KeyError as e:
print 'Key error occured %s' % e
def index(self):
""" The main page shown for sensor, lists all activated sensors, grouped by location. """
location_list = Session.query(SensorLocation)\
.join(Sensor)\
.filter(Sensor.active == 1)\
.all()
locations = Session.query(SensorLocation).all()
return ({ 'page_title': 'Sensor', 'secondmenu': self._create_sidemenu(),
'location_list' : location_list, 'locations' : locations})
def get_object(self, id=0):
""" Return policy object as a JSON string.
Produces slightly different output than get_policy_item().
"""
if id == 0:
policy_object = Session.query(PolicyObject).first()
else:
policy_object = Session.query(PolicyObject).filter(
PolicyObject.id == id).one()
return simplejson.dumps(policy_object.make_json())
def add_policy(self, name='', description=''):
""" Creates a new policy based on user input.
Known bug: Doesn't check exsitence of policy.
"""
try:
self.check_status()
if len(name) > 0 and len(description) > 0:
new_policy = PolicyChainMeta(escape_string(name.capitalize()), escape_string(description))
Session.add(new_policy)
Session.flush()
except SystemLockedException:
pass
finally:
raise cherrypy.HTTPRedirect("/policy/") # Return to index
def index(self):
""" Returns a list of sources.
Landing page for Snort rule sources, returns a list of all the sources registered in the system.
"""
sources = Session.query(UpdateSource)
return (self.render({ 'page_title': 'Rules', 'sources': sources }))
def source(self, id = 0):
""" Returns interface to add Snort sources. """
source_url = None
if id is not 0:
source_url = Session.query(UpdateSource).get(id)
return { 'page_title': 'Rules', 'source_url': source_url}
def choose_object(self, **kwargs):
""" Choose a excisting policy object. """
try:
if 'object-id' not in kwargs and 'policy-id' not in kwargs:
raise KeyError
object_id = kwargs['object-id']
policy_id = kwargs['policy-id']
policy_object = Session.query(PolicyChain).filter(
PolicyChain.id == policy_id).one()
policy_object.policyobject_id = object_id
Session.merge(policy_object)
Session.flush()
except KeyError:
print 'KeyError'
def delete_policy(self, id = 0):
""" Sets a policy to deactive in the database. """
try:
self.check_status()
except SystemLockedException:
raise cherrypy.HTTPRedirect('/policy/')
if id == 0:
raise NoResultFound
try:
policy = Session.query(PolicyChainMeta).filter(PolicyChainMeta.id == id).one()
policy.active = 0
Session.merge(policy)
Session.flush()
except NoResultFound:
pass
finally:
raise cherrypy.HTTPRedirect('/policy/')
def add_location(self, **kwargs):
""" Adds a new location to the database """
try:
name = escape_string(kwargs['location_name'])
if len(name) == 0:
raise TypeError('The location name is empty!')
new_location = SensorLocation(name)
Session.add(new_location)
Session.flush()
print new_location
return simplejson.dumps(new_location.id)
except KeyError as e:
print 'KEYERROR; this key does not excist: %s' % s
except TypeError as e:
print e
def get_policy_item(self, id=0):
""" Return a policy item as a JSON string. """
cherrypy.response.headers['Content-Type'] = 'application/json'
try:
policy_item = Session.query(PolicyChain).join(PolicyObject).filter(
PolicyChain.id == id).one()
policy_object = policy_item.policyobject
except NoResultFound:
return simplejson.dumps({'id' : 0, 'active' : 0, 'type' : 0, 'contents' : '' })
return simplejson.dumps(policy_object.make_json())
def index(self):
""" Returns a list of the active policies in the system.
Provides a list of active policies and the ability to create more.
"""
policy_chain = Session.query(PolicyChainMeta).filter(PolicyChainMeta.active == 1).all()
for item in policy_chain:
if item.name is None:
item.name = 'N/A'
return self.render({ 'page_title': 'Policy chains', 'policy_chain': policy_chain})
def register_file_in_database(self, file, urlid):
""" Register a Snort rule in the database
This method will register a Snort rule file with its full path and file name.
:param file: the full path with filename.
:param urlid: the update source. Used to connect registered file to correct source in the database.
"""
try:
base_path = str(self.tempDownloadFolder)
search_pattern = re.compile(r"(?P<path>[A-z]+)/(?P<name>(\w|\.|-)+)$") # Search for file ending
file_attributes = search_pattern.search(file) #
file_name = file_attributes.group("name")
file_path = file_attributes.group("path")
try:
file_from_db = (
Session.query(UpdateFile)
.filter(and_(UpdateFile.name == file_name, UpdateFile.updatesource == urlid))
.one()
)
if file_from_db is None:
raise NoResulFound
else:
return file_from_db.id
except NoResultFound:
new_file = UpdateFile(file_name, urlid, file_path)
Session.add(new_file)
Session.flush()
return new_file.id
except ImportError:
# print('') #TODO: TRANSLATe
# raise
# exit(1)
print e
exit()
return 0
def get_ignore_list(self, urlID):
""" Gets the list of files to ignore """
filesToIgnore = [] # List of files and folders to ignore
try:
IgnoreList = Session.query(UpdateFilter).filter(UpdateFilter.updateurl == urlID)
for item in IgnoreList:
filesToIgnore.append(item.path)
except Exception as e:
print ("Exception; problem with generation of ignore list")
print e
exit(1)
return filesToIgnore
def download_rules(self):
""" Downloads all the rules from a provided array of lists and reads from file """
# PROPER VALUES
tempDownloadFolder = self.tempDownloadFolder
try:
listOfUrl = Session.query(UpdateSource).filter(
UpdateSource.active == 1
) # sql.execute("SELECT id, url FROM UpdateUrl WHERE active = 1")
for listUrl in listOfUrl:
try:
# Get ignores
self.register_update_in_database(listUrl.id) # Register
IgnoreList = self.get_ignore_list(listUrl.id)
FileUrl = listUrl.url # TODO: DELETE
urlHandler = urllib2.urlopen(FileUrl)
localFileName = re.search(r"(?<=/{1})[\w\-\.]+\.(rules|tar\.gz|zip|conf)", FileUrl)
localFilePath = os.path.join(tempDownloadFolder, os.path.basename(localFileName.group(0)))
localFile = open(localFilePath, "w")
localFile.write(urlHandler.read())
localFile.close()
isArchive = self.check_if_file_is_archive(localFilePath)
if isArchive == None:
self.read_rule_file(localFilePath, listUrl.id, IgnoreList) # Traverse dir
else:
for file in isArchive:
full_path = os.path.join(tempDownloadFolder, file)
# print 'Reading file name %s' % full_path
self.read_rule_file(full_path, listUrl.id, IgnoreList)
# self.get_rule_contents(os.path.join(tempDownloadFolder, file), listUrl.id, IgnoreList)
except urllib2.HTTPError, e:
errorCodes = {404: "Not Found", 403: "Forbidden"}
print r"""
===================
Error while downloading updates
Url: %s
Error code: %i
Error message: %s
===================
""" % (
FileUrl,
e.code,
errorCodes[e.code],
)
except urllib2.URLError, e:
print "Error with URL", e.reason # TODO: OVERSETT
def get_sensor_data(self, id):
""" Returns a JSON string containing a sensors data.
This function takes a sensors ID as paramater, and providing it exists will return all its data as JSON string.
The function is used by AJAX to populate forms. """
try:
if not id.isdigit():
raise TypeError
sensor = Session.query(Sensor).filter(Sensor.id == id).one()
except TypeError:
print 'Type error occured'
except NoResultFound:
print 'No result found'
return simplejson.dumps(sensor.make_json())
def edit_policy(self, id = 0):
""" Returns a interface to edit entire policies.
Returns a list of the contents of the policy, correctly ordered. """
if id == 0:
raise cherrypy.HTTPRedirect("/policy/") # Return to index
# Get information about policy
chain_contents = [] # List of policy chain objects
policy_info = Session.query(PolicyChainMeta).filter(PolicyChainMeta.id == id).one()
policyContents = Session.query(PolicyChain).join(PolicyObject).filter(
and_(PolicyChain.chain_id == id, PolicyChain.parent == 0)).first()
if policyContents is None:
next = 0
policyContents = Session.query(PolicyChain).filter(
and_(PolicyChain.chain_id == id, PolicyChain.parent == 0)).first()
if policyContents is not None:
chain_contents.append({'id': policyContents.id, 'object': None})
else:
chain_contents.append({'id': policyContents.id, 'object': policyContents.policyobject})
next = int(policyContents.child)
while next is not 0:
cp = Session.query(PolicyChain).join(PolicyObject).filter(PolicyChain.id == next).first()
if cp is not None:
chain_contents.append({'id': cp.id, 'object': cp.policyobject})
next = int(cp.child)
else:
cp = Session.query(PolicyChain).filter(PolicyChain.id == next).first()
chain_contents.append({'id': cp.id, 'object': None})
next = 0
all_objects = Session.query(PolicyObject).all()
return self.render({'page_title' : 'Edit policy',
'all_objects' : all_objects,
'policy' : {'contents' : chain_contents, 'info' : policy_info}})
def __init__(self):
""" Create temporary folder and get all the rule files """
self.temp_sensor_folder = mkdtemp()
self.rule_files = Session.query(UpdateFile)#.join(PolicyObject).filter(and_(PolicyChain.chain_id == chainId, PolicyChain.parent == 0))
def deactivate_rule(self, rule):
""" Deactivates and merges rules. Also returns the old rule to """
rule.active = 0
Session.merge(rule)
return rule
def index(self):
""" Returns a list of the 10 last log events. """
event_content = Session.query(EventLog).limit(10)
return (self.render({ 'page_title': 'Main page', 'event_content': event_content}))
def addsensorindex(self):
""" Add a new sensor to the database"""
locations = Session.query(SensorLocation).all()
return ({ 'page_title': 'Add Sensor', 'secondmenu': self._create_sidemenu(), 'locations' : locations})
def remove_object(self, **kwargs):
""" Remove the object from the list """
if 'Referer' in cherrypy.request.headers:
return_url = cherrypy.request.headers['Referer']
else:
return_url = '/policy/'
try:
if kwargs['object-id'] is None or not kwargs['object-id'].isdigit():
raise TypeError('Policy ID is either None or ID is not int')
if kwargs['policy-id'] is None:
raise TypeError('Policy content None')
object_id = int(kwargs['object-id'])
policy_id = int(kwargs['policy-id'])
delete_object = Session.query(PolicyChain).filter(and_(
PolicyChain.chain_id == policy_id, PolicyChain.id == object_id)).one()
parent = delete_object.parent
child = delete_object.child
Session.delete(delete_object)
if parent != 0:
parent_object = Session.query(PolicyChain).filter(and_(
PolicyChain.id == parent, PolicyChain.chain_id == policy_id)).one()
parent_object.child = child
Session.merge(parent_object)
if child != 0:
child_object = Session.query(PolicyChain).filter(and_(
PolicyChain.id == child, PolicyChain.chain_id == policy_id)).one()
child_object.parent = parent
Session.merge(child_object)
Session.flush()
raise cherrypy.HTTPRedirect(return_url)
except NoResultFound:
raise cherrypy.HTTPRedirect(return_url)
except KeyError as e:
raise
def producePolicyChain(self):
""" Produces configuration file based on contents in policy chain
The job is done in the following way
#. Compile a list of all the sensor grouped by the policy chain they use
#. Create a list of policy contents."""
chainList = {} # The list of sensors grouped by policy chain
try:
""" Compiles the list of sensors and policy chains. """
for sensor in Session.query(Sensor).all():
if str(sensor.policychain) not in chainList: # If it isn't already in the list
chainList[str(sensor.policychain)] = []
chainList[str(sensor.policychain)].append(sensor.ip)
except KeyError as e:
print 'Error with key\n', e
for ipList in chainList.items():
chainId = ipList[0]
chain_contents = []
PolicyFile = {PREPEND : [],
CONTENTS : [],
APPEND : []}
policy_contents = Session.query(PolicyChain).join(PolicyObject).filter(and_(PolicyChain.chain_id == chainId, PolicyChain.parent == 0)).first()
if policy_contents is not None:
chain_contents.append({'id': policy_contents.id,'object': policy_contents.policyobject})
next = int(policy_contents.child)
else:
next = 0
while next is not 0:
cp = Session.query(PolicyChain).join(PolicyObject).filter(PolicyChain.id == next).first()
if cp is not None: # If the chain element doesn't have a Object connected to it
chain_contents.append({'id': cp.id, 'object': cp.policyobject})
else:
cp = Session.query(PolicyChain).filter(PolicyChain.id == next).first()
next = int(cp.child)
for element in chain_contents:
policy_type = element['object'].type
strings = element['object'].contents.splitlines()
for string in strings:
if not string.startswith('#'): # Check if a line is really a comment
PolicyFile[policy_type].append(string) # Add to the proper list
#Build policychain
configString = ''
inclusion = {'file': [], 'sid':[], 'regex': []}
exclusion = {'file': [], 'sid':[]}
rewrite = {}
action_package = {'enable' : inclusion, 'disable' : exclusion, 'rewrite' : rewrite}
pfinder = re.compile(r'^(?P<action>(disable|enable|set|rewrite))\s+(?P<target>(file|sid|regex)?)\s+(?P<content>(.+)?)$')
for policy_string in PolicyFile[CONTENTS]:
p_string_contents = pfinder.search(policy_string) # Builds object from content
if p_string_contents is not None:
policy_action = p_string_contents.group('action')
policy_target = p_string_contents.group('target')
policy_content = p_string_contents.group('content')
if policy_action == 'rewrite':
find_sid_pattern = re.compile(r'^\d+')
rewrite_sid = find_sid_pattern.search(policy_content)
action_package[policy_action][rewrite_sid.group(0)] = policy_content
else:
if policy_action == 'enable' and policy_target != 'regex':
try:
action_package['disable'][policy_target].remove(policy_content)
except ValueError:
pass
elif policy_action == 'disable':
try:
action_package['enable'][policy_target].remove(policy_content)
except ValueError:
pass
action_package[policy_action][policy_target].append(policy_content)
# Add content to file
for line in PolicyFile[PREPEND]: # Write all prepend content to file
configString += line + '\n'
for file in self.rule_files:
if len(exclusion['file']) is not 0:
if file.name not in exclusion['file']:
rules = Session.query(Rule).filter(Rule.file == file.id)
for rule in rules:
if str(rule.sid) not in exclusion['sid'] and rule.active is not 0:
if rule.sid in rewrite:
configstring += self.prepare_rule(rule.rule,rewrite[rule.sid])
else:
configString += rule.rule
if len(inclusion['file']) is not 0:
if file.name in inclusion['file']:
rules = Session.query(Rule).filter(Rule.file == file.id)
for rule in rules:
if str(rule.sid) not in exclusion['sid'] and rule.active is not 0:
if str(rule.sid) == rewrite:
configstring += self.prepare_rule(rule.rule, rewrite[rule.sid])
else:
configString += rule.rule
if len(inclusion['sid']) is not 0:
for inc_sid in inclusion['sid']:
try:
rule = Session.query(Rule).filter(Rule.sid == inc_sid).order_by(Rule.rev.desc()).first()
if rule is None:
raise NoResultFound
if str(rule.sid) in rewrite:
configString += self.prepare_rule(rule.rule, rewrite[str(rule.sid)])
else:
configString += rule.rule
except NoResultFound:
pass
# Enable from regex
if len(inclusion['regex']) > 0: # Activates rules based on regex pattern
""" Will only fetch all the rules __IF__ there's regex item """
self.all_the_rules = Session.query(Rule).filter(Rule.active == 1).all() # Get ALL the rules
for sid_expression in inclusion['regex']: # Go through each pattern to search
reg_pattern = re.compile(r'%s' % sid_expression)
for rule in self.all_the_rules: # Go through rules
if reg_pattern.search(rule.rule):
if (str(rule.sid) not in inclusion['sid'] and
self.rule_files[rule.file].name not in inclusion['file']):
configString += rule.rule
for line in PolicyFile[APPEND]: # Write all the append content to file
configString += line + '\n'
for sensor in ipList[1]:
#roll out policychain
try:
f = open(os.path.join(self.temp_sensor_folder, sensor + '.txt'),'w')
f.write(configString)
f.close()
except IOError:
print 'IOError'
def new_object(self, id = 0):
""" Adds a new object to the end of the policy """
if 'Referer' in cherrypy.request.headers:
return_url = cherrypy.request.headers['Referer']
else:
return_url = '/policy/'
try:
last_object = Session.query(PolicyChain).filter(and_(
PolicyChain.chain_id == id, PolicyChain.child == 0)).one();
parent = last_object.id
new_object = PolicyChain(id, 0, parent, 0)
Session.add(new_object)
Session.flush()
last_object.child = new_object.id
Session.merge(last_object)
Session.flush()
except NoResultFound:
new_object = PolicyChain(id, 0, 0, 0)
Session.add(new_object)
Session.flush()
raise cherrypy.HTTPRedirect(return_url)