def insert_websites_to_mongo(website_list):
"""
Function to insert a list of website objects into mongodb
"""
mongo = WebsiteMongo()
for w in website_list:
print("Inserting", w.url, "into mongodb")
mongo.create_website(w)
def insert_websites_to_mongo(website_list):
"""
Function to insert a list of website objects into mongodb
"""
mongo = WebsiteMongo()
for w in website_list:
print("Inserting", w.url, "into mongodb")
mongo.create_website(w)
def insert_website_to_mongo(website):
"""
Function to insert a list of website objects into mongodb
"""
try:
mongo = WebsiteMongo()
mongo.create_website(website)
except:
print("[Robly] Error inserting {} into mongodb".format(website.url))
def test_search_website(self):
#Uses Full Text Search
#MongoDB server must be started with command "--setParameter textSearchEnabled=true"
#in order for FTS to be enabled
mongo = WebsiteMongo()
query = "git"
websites_list, stats_obj = mongo.search_websites(query)
self.assertIsNotNone(websites_list)
self.assertIsNotNone(stats_obj)
def test_search_website(self):
#Uses Full Text Search
#MongoDB server must be started with command "--setParameter textSearchEnabled=true"
#in order for FTS to be enabled
mongo = WebsiteMongo()
query = "git"
websites_list, stats_obj = mongo.search_websites(query)
self.assertIsNotNone(websites_list)
self.assertIsNotNone(stats_obj)
def insert_website_to_mongo(website):
"""
Function to insert a list of website objects into mongodb
"""
try:
mongo = WebsiteMongo()
mongo.create_website(website)
except:
print("[Robly] Error inserting {} into mongodb".format(website.url))
def search():
DEBUG_INFO = "[ROBLY] webapp.py - /search - "
#logging.debug(DEBUG_INFO + "POST request = " + request.form)
query = request.form['search_box']
query.strip()
logging.debug(DEBUG_INFO + "query = " + query)
#Create query parser object with query string provided by user
query_parser = QueryParser(query)
search_query, search_context = query_parser.extract_context_and_search_query(
)
if len(query) > 1:
logging.debug(DEBUG_INFO + "SearchQuery = " + search_query)
logging.debug(DEBUG_INFO + "SearchContext = " + search_context)
#query should now have been pruned of unnecessary words and characters
#Get info from database
try:
mongo = WebsiteMongo()
logging.debug(
DEBUG_INFO +
"Attempthing to connect with mongodb searchquery='{}' "
"context=''".format(search_query, search_context))
websites, stats = mongo.search_websites(search_query,
search_context)
#convert microseconds to seconds
seconds = stats.time_micros / 1000000
#Use duckduckgo api if no results found
if len(websites) < 1:
websites = get_websites_from_duckduckgo(search_query)
else:
#Add page rank to full text search score
try:
for w in websites:
w.score += w.pagerank
except:
print(
"[ROBLY] Problem calculating search result score with page rank"
)
#Sort list of websites by score
from operator import attrgetter
websites.sort(key=attrgetter('score'), reverse=False)
return render_template('search_results.html',
search_results=websites,
stats=stats,
seconds=seconds)
except Exception as e:
logging.error(
DEBUG_INFO +
"Error searching mongodb with the searchquery '{} - {}'".
format(search_query, str(e)))
return redirect(url_for('index'))
else:
return redirect(url_for('index'))
def search():
DEBUG_INFO = "[ROBLY] webapp.py - /search - "
#logging.debug(DEBUG_INFO + "POST request = " + request.form)
query = request.form['search_box']
query.strip()
logging.debug(DEBUG_INFO + "query = " + query)
#Create query parser object with query string provided by user
query_parser = QueryParser(query)
search_query, search_context = query_parser.extract_context_and_search_query()
if len(query) > 1:
logging.debug(DEBUG_INFO + "SearchQuery = " + search_query)
logging.debug(DEBUG_INFO + "SearchContext = " + search_context)
#query should now have been pruned of unnecessary words and characters
#Get info from database
try:
mongo = WebsiteMongo()
logging.debug(DEBUG_INFO + "Attempthing to connect with mongodb searchquery='{}' "
"context=''".format(search_query, search_context))
websites, stats = mongo.search_websites(search_query, search_context)
#convert microseconds to seconds
seconds = stats.time_micros / 1000000
#Use duckduckgo api if no results found
if len(websites) < 1:
websites = get_websites_from_duckduckgo(search_query)
else:
#Add page rank to full text search score
try:
for w in websites:
w.score += w.pagerank
except:
print("[ROBLY] Problem calculating search result score with page rank")
#Sort list of websites by score
from operator import attrgetter
websites.sort(key=attrgetter('score'), reverse=False)
return render_template('search_results.html', search_results=websites, stats=stats, seconds=seconds)
except Exception as e:
logging.error(DEBUG_INFO + "Error searching mongodb with the searchquery '{} - {}'".format(search_query,
str(e)))
return redirect(url_for('index'))
else:
return redirect(url_for('index'))
def test_create_insert_website(self):
mongo = WebsiteMongo()
website = generate_website()
successful = mongo.create_website(website)
print("blah")
def test_create_insert_website(self):
mongo = WebsiteMongo()
website = generate_website()
successful = mongo.create_website(website)
print("blah")
