def jsonGetWordClouds(self):
     """
         Endpoint for the AJAX request to get the search terms in the word cloud.
         Calls `Filter.onlineSearches()` in :doc:`filters` for each search engine in the config file.
         Returns a dictionary of terms.
     """
     highlight_funcs, remove_funcs = convertFilters(self.request.args)            
     amount = self.request.args.get('amount', 20)
     
     search_results = {}
     opts = config.options('search_engines')
     for opt in opts:
         cloud, terms, unique, small = Filter.onlineSearches(opt, highlight_funcs, remove_funcs,
                                                             amount)
         search_results[opt] = (cloud, terms, opt, config.get('search', opt), unique, small)
     return search_results
 def addEntry(self, program, file, group):
     """
         Calls the generator `convert_file()` found in :doc:`converters` on each row of the file, 
         and adds the result to the database. If an exception happens during the converting and
         adding of data, then the session is rolled back and `None` is returned. Otherwise
         `True` is returned. 
         
         .. note::
             This had been optimised to make the adding of data as fast as possible, but
             has been slowed down again by adding search terms. 
             
             **ToDo**: Optimise the adding of search terms. 
     """
     session.flush()
     browser_ids = {}        
     
     try:
         entry_ins = Entry.__table__.insert()
         url_ins = URL.__table__.insert()
         count_num = 0
         for d in convert_file(program, file):
             
             browser_name = d.pop('browser_name')
             browser_version = d.pop('browser_version')
             source = d.pop('source_file')
                             
             key = (browser_name, browser_version, source)                
             browser_id = browser_ids.get(key)
             if browser_id is None:
                 browser = Browser.getFilterBy(name=browser_name, version=browser_version,
                                               source=source).first()
                 if browser is None:
                     browser = Browser(*key)
                     session.add(browser)
                     session.flush()
                 browser_id = browser_ids[key] = browser.id
             
             # optimised to make adding data as fast as possible - ignores the ORM
             v = d.pop('access_time')
             if v is not None:
                 d['access_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0)
                 d['access_time'] = time(v.hour, v.minute, v.second, v.microsecond)
             else:
                 continue # don't add data without an access time
             v = d.pop('modified_time')
             if v is not None:
                 d['modified_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0)
                 d['modified_time'] = time(v.hour, v.minute, v.second, v.microsecond)
             else:
                 d['modified_date'] = None
                 d['modified_time'] = None
             
             result = session.execute(entry_ins.values(browser_id=browser_id, 
                                                       group_id=group.id,
                                                        **d))                
             entry_id = result.last_inserted_ids()[0]
             
             # add URLS
             url = URL(d['url'])
             session.execute(url_ins.values(entry_id=entry_id, **url.asDict()))  
             url_id = result.last_inserted_ids()[0]
             
             # add search terms
             # TODO: make this optimised like above!
             entry = Entry.get(entry_id)
             url = URL.get(url_id)
             
             opts = config.options('search_engines')
             if url.query != None and 'search' in url.path:
                 for opt in opts:
                     if opt in url.netloc:
                         query = url.query.split(config.get('search_engines', opt)+'=')\
                                 [-1].split('&')[0]
                         q_string, terms = SearchTerms.getTerms(urllib.unquote(query))
                         url.search = q_string 
                         for term in terms:
                             t = SearchTerms.getFilterBy(term=term, engine=opt).first()
                             if t is None:
                                 t = SearchTerms(term, opt, config.get('search', opt))
                                 session.add(t)
                             else:
                                 t.occurrence = t.occurrence + 1
                             entry.search_terms.append(t)
                             session.flush()   
             count_num = count_num + 1
         if count_num == 0:
         # we have not added anything, but no exceptions where raised.
             return None, "No entries found in the uploaded file"             
     except Exception, e:
         session.rollback()            
         return None, e