def insert_websites_to_mongo(website_list): """ Function to insert a list of website objects into mongodb """ mongo = WebsiteMongo() for w in website_list: print("Inserting", w.url, "into mongodb") mongo.create_website(w)
def insert_website_to_mongo(website): """ Function to insert a list of website objects into mongodb """ try: mongo = WebsiteMongo() mongo.create_website(website) except: print("[Robly] Error inserting {} into mongodb".format(website.url))
def test_search_website(self): #Uses Full Text Search #MongoDB server must be started with command "--setParameter textSearchEnabled=true" #in order for FTS to be enabled mongo = WebsiteMongo() query = "git" websites_list, stats_obj = mongo.search_websites(query) self.assertIsNotNone(websites_list) self.assertIsNotNone(stats_obj)
def search(): DEBUG_INFO = "[ROBLY] webapp.py - /search - " #logging.debug(DEBUG_INFO + "POST request = " + request.form) query = request.form['search_box'] query.strip() logging.debug(DEBUG_INFO + "query = " + query) #Create query parser object with query string provided by user query_parser = QueryParser(query) search_query, search_context = query_parser.extract_context_and_search_query( ) if len(query) > 1: logging.debug(DEBUG_INFO + "SearchQuery = " + search_query) logging.debug(DEBUG_INFO + "SearchContext = " + search_context) #query should now have been pruned of unnecessary words and characters #Get info from database try: mongo = WebsiteMongo() logging.debug( DEBUG_INFO + "Attempthing to connect with mongodb searchquery='{}' " "context=''".format(search_query, search_context)) websites, stats = mongo.search_websites(search_query, search_context) #convert microseconds to seconds seconds = stats.time_micros / 1000000 #Use duckduckgo api if no results found if len(websites) < 1: websites = get_websites_from_duckduckgo(search_query) else: #Add page rank to full text search score try: for w in websites: w.score += w.pagerank except: print( "[ROBLY] Problem calculating search result score with page rank" ) #Sort list of websites by score from operator import attrgetter websites.sort(key=attrgetter('score'), reverse=False) return render_template('search_results.html', search_results=websites, stats=stats, seconds=seconds) except Exception as e: logging.error( DEBUG_INFO + "Error searching mongodb with the searchquery '{} - {}'". format(search_query, str(e))) return redirect(url_for('index')) else: return redirect(url_for('index'))
def search(): DEBUG_INFO = "[ROBLY] webapp.py - /search - " #logging.debug(DEBUG_INFO + "POST request = " + request.form) query = request.form['search_box'] query.strip() logging.debug(DEBUG_INFO + "query = " + query) #Create query parser object with query string provided by user query_parser = QueryParser(query) search_query, search_context = query_parser.extract_context_and_search_query() if len(query) > 1: logging.debug(DEBUG_INFO + "SearchQuery = " + search_query) logging.debug(DEBUG_INFO + "SearchContext = " + search_context) #query should now have been pruned of unnecessary words and characters #Get info from database try: mongo = WebsiteMongo() logging.debug(DEBUG_INFO + "Attempthing to connect with mongodb searchquery='{}' " "context=''".format(search_query, search_context)) websites, stats = mongo.search_websites(search_query, search_context) #convert microseconds to seconds seconds = stats.time_micros / 1000000 #Use duckduckgo api if no results found if len(websites) < 1: websites = get_websites_from_duckduckgo(search_query) else: #Add page rank to full text search score try: for w in websites: w.score += w.pagerank except: print("[ROBLY] Problem calculating search result score with page rank") #Sort list of websites by score from operator import attrgetter websites.sort(key=attrgetter('score'), reverse=False) return render_template('search_results.html', search_results=websites, stats=stats, seconds=seconds) except Exception as e: logging.error(DEBUG_INFO + "Error searching mongodb with the searchquery '{} - {}'".format(search_query, str(e))) return redirect(url_for('index')) else: return redirect(url_for('index'))
def test_create_insert_website(self): mongo = WebsiteMongo() website = generate_website() successful = mongo.create_website(website) print("blah")