def jsonAddFilter(self): """ Endpoint for the AJAX request to validate and add a new filter. If valid, the filter is added and a new hash is created and appended to the database file with the current date and time, returning `True`. Otherwise `self.form_error` is returned. """ if self.validate_form(add_filter_form()): text = self.form_result['name'] label = "".join(self.form_result['name'].split(' ')).lower() fq = FilterQuery() fil = Filter(label, text) for f in self.form_result['filter']: cls = f['cls'] attr = f['attribute'] func = f['function'] val = f['value'] if f['value_list'] is not None: val_list = f['value_list'] else: val_list = None fq.add_element(cls, attr, func, val, val_list) fil.query = fq session.add(fil) session.commit() self.write_log(self.dbfile, 'Added a filter called ' + text) return True else: return self.form_error
def addData(self, entry): """ Given a validated form called `entry`, adds the group to the database, then calls `self.addEntry()` to convert the data in `entry['data']` to `Entry` objects. Returns the group if the adding of entries was successful, and otherwise returns `None`. """ group = Group(entry[u'name'], entry[u'desc'], self.case, get_program(entry[u'program'])) group.csv_name = entry[u'data'].filename session.add(group) # convert the file to entries done, error_msg = self.addEntry(group.program, entry['data'].stream, group) if done == True: return group, None # used in jsonEditEntries else: return None, error_msg # exception happened!
def jsonAddCase(self): """ Endpoint for wizard 1 form (adding case name and dbfile name). This is done via Ajax. If the form has errors, the error dictionary `self.form_errors` is jsonified and passed back to the webpage and the errors displayed to the user. If the form is valid, then the new case is created, the database initialised and data saved to the database. This will then return `True`. """ if self.validate_form(wizard1_form()): # form is validated, so add the case information Case.create_database(self.form_result[u'dbfile']) # switch to the new database case = Case(self.form_result[u'name']) session.add(case) self.dbfile = self.form_result[u'dbfile'] #set sqlite database location cookie return True else: return self.form_error
def addEntry(self, program, file, group): """ Calls the generator `convert_file()` found in :doc:`converters` on each row of the file, and adds the result to the database. If an exception happens during the converting and adding of data, then the session is rolled back and `None` is returned. Otherwise `True` is returned. .. note:: This had been optimised to make the adding of data as fast as possible, but has been slowed down again by adding search terms. **ToDo**: Optimise the adding of search terms. """ session.flush() browser_ids = {} try: entry_ins = Entry.__table__.insert() url_ins = URL.__table__.insert() count_num = 0 for d in convert_file(program, file): browser_name = d.pop('browser_name') browser_version = d.pop('browser_version') source = d.pop('source_file') key = (browser_name, browser_version, source) browser_id = browser_ids.get(key) if browser_id is None: browser = Browser.getFilterBy(name=browser_name, version=browser_version, source=source).first() if browser is None: browser = Browser(*key) session.add(browser) session.flush() browser_id = browser_ids[key] = browser.id # optimised to make adding data as fast as possible - ignores the ORM v = d.pop('access_time') if v is not None: d['access_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0) d['access_time'] = time(v.hour, v.minute, v.second, v.microsecond) else: continue # don't add data without an access time v = d.pop('modified_time') if v is not None: d['modified_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0) d['modified_time'] = time(v.hour, v.minute, v.second, v.microsecond) else: d['modified_date'] = None d['modified_time'] = None result = session.execute(entry_ins.values(browser_id=browser_id, group_id=group.id, **d)) entry_id = result.last_inserted_ids()[0] # add URLS url = URL(d['url']) session.execute(url_ins.values(entry_id=entry_id, **url.asDict())) url_id = result.last_inserted_ids()[0] # add search terms # TODO: make this optimised like above! entry = Entry.get(entry_id) url = URL.get(url_id) opts = config.options('search_engines') if url.query != None and 'search' in url.path: for opt in opts: if opt in url.netloc: query = url.query.split(config.get('search_engines', opt)+'=')\ [-1].split('&')[0] q_string, terms = SearchTerms.getTerms(urllib.unquote(query)) url.search = q_string for term in terms: t = SearchTerms.getFilterBy(term=term, engine=opt).first() if t is None: t = SearchTerms(term, opt, config.get('search', opt)) session.add(t) else: t.occurrence = t.occurrence + 1 entry.search_terms.append(t) session.flush() count_num = count_num + 1 if count_num == 0: # we have not added anything, but no exceptions where raised. return None, "No entries found in the uploaded file" except Exception, e: session.rollback() return None, e
def addDefaultFilters(self): """ Adds the default filters for the timegraph such as filtering by browser type, group, work hours, Google searches and local files. Gets called when a new case is being set up in `finish_wizard()` in :doc:`caseController`. """ # Add filters for the browsers available, unless only one browser, then a filter on # everything is pointless browsers = Browser.getAll().group_by(Browser.name).all() if len(browsers) > 1: for browser in browsers: f = Filter(u''.join(browser.name.lower().split(' ')), browser.name) fq = FilterQuery() cls = u'Browser' attr = u'name' func = u'Is' val = browser.name fq.add_element(cls, attr, func, val, None) f.query = fq session.add(f) session.flush() # filters for Google searches f = Filter(u'googlesearch', u'Google searches') fq = FilterQuery() params = [(u'URL Parts', u'query', u'Is not', None, None), (u'URL Parts', u'netloc', u'Is not', None, None), (u'URL Parts', u'path', u'Is not', None, None), (u'URL Parts', u'netloc', u'Contains', u'google', None), (u'URL Parts', u'path', u'Contains', u'search', None), ] for entry in params: fq.add_element(*entry) f.query = fq session.add(f) session.flush() # filters for local files accessed files = URL.getFilterBy(scheme="file").all() if files is not None: f = Filter(u'files', u'Local Files') fq = FilterQuery() cls = u'URL Parts' attr = u'scheme' func = u'Is' val = u'file' fq.add_element(cls, attr, func, val, None) f.query = fq session.add(f) session.flush() # filters for different groups groups = Group.getAll().all() if len(groups) > 1: for group in groups: f = Filter(u''.join(group.name.lower().split(' ')), group.name) fq = FilterQuery() cls = u'Group' attr = u'name' func = u'Is' val = group.name fq.add_element(cls, attr, func, val, None) f.query = fq session.add(f) session.flush() # filters for work hours f = Filter(u'workhours', u'Work hours') fq = FilterQuery() five = time(17, 00, 01) nine = time(8, 59, 59) params = [(u'Entry', u'access_time', u'Less than', five, None), (u'Entry', u'access_time', u'Greater than', nine, None), ] for entry in params: fq.add_element(*entry) f.query = fq session.add(f) session.flush() # filters for adverts f = Filter(u'adverts', u'Advert URLs') fq = FilterQuery() fq.add_element(u'URL Parts',u'domain',u'Is not', None, None) fq.add_element(u'URL Parts',u'domain',u'Is in list', None, 'advert_domainnames.txt') f.query = fq session.add(f) session.flush() # filters for Facebook, MySpace, Bebo, twitter, hi5 f = Filter(u'social', u'Social Networking URLs') fq = FilterQuery() fq.add_element(u'URL Parts',u'domain',u'Is not', None, None) fq.add_element(u'URL Parts',u'domain',u'Is in list', None, 'socialmedia.txt') f.query = fq session.add(f) session.flush() # filters for email f = Filter(u'email', u'Web Email') fq = FilterQuery() five = time(17, 00, 01) nine = time(8, 59, 59) params = [(u'Entry', u'url', u'Contains', 'mail', None), (u'URL Parts', u'scheme', u'Is Not', 'file', None), ] for entry in params: fq.add_element(*entry) f.query = fq session.add(f) session.flush() # filters for news f = Filter(u'news', u'News URLs') fq = FilterQuery() fq.add_element(u'URL Parts',u'hostname',u'Is not', None, None) fq.add_element(u'URL Parts',u'hostname',u'Is in list', None, 'news.txt') f.query = fq session.add(f) session.flush()