def search(request):
vars = {}
if request.GET.get('query'):
#form was submitted
page = int(request.GET.get('page', 1))
search_form = SearchForm(request.GET)
query = search_form['query'].value().replace('+', ' AND ').replace(' -', ' NOT ')
profile_ids = search_form['profiles'].value()
if profile_ids and len(profile_ids) != len(search_form['profiles'].field.choices):
profiles = SpiderProfile.objects.filter(active=True).filter(pk__in=profile_ids)
query = "(%s) AND (%s)" % (query, ' OR '.join(['site:%s' % p.name for p in profiles] ))
page_index = WhooshPageIndex()
start_time = datetime.now()
results = page_index.search(query, pagenum=page, pagelen=100)
query_time = datetime.now() - start_time
query_time_formatted = float(query_time.seconds) + float(query_time.microseconds)/1000000.0
vars['query_time'] = query_time_formatted
vars['page'] = page
vars['pagecount'] = results.pagecount
vars['results'] = [ dict(title=r['title'], highlight=r.highlights('content'), url=r['url']) for r in results]
vars['pagination_results'] = range(1, len(results))
vars['num_results'] = len(results)
template = 'search_results.html'
else:
search_form = SearchForm()
template = 'index.html'
vars['search_form'] = search_form
return render(request, template, vars)
def spider(profile, log=True):
depth = profile.depth
indexer = WhooshPageIndex()
pending_urls = Queue.Queue()
processed_responses = Queue.Queue(maxsize=50)
finished = threading.Event()
visited = {}
scheduled = set()
thread_count = profile.threads or getattr(settings, 'SPIDER_THREADS', 4)
threads = [SpiderThread(pending_urls, processed_responses, finished, profile) for _ in range(thread_count)]
pending_urls.put((profile.base_url, '', depth))
scheduled.add(profile.base_url)
extractor = profile.get_extractor_class()
[t.start() for t in threads]
processed_url = None
try:
while 1:
try:
# pull an item from the response queue
result_dict, urls, depth = processed_responses.get(timeout=profile.timeout)
except Queue.Empty:
#check to see if any of the workers are still doing anything
done = True
for t in threads:
if t.working:
print "Thread %s is still working, not exiting" % t
done = False
if done:
print "All threads done working"
finished.set()
break
else:
# save the result
processed_url = result_dict['url']
raw_content = result_dict['content']
unicode_content = BeautifulSoup.UnicodeDammit(raw_content, isHTML=True).unicode
e = extractor(unicode_content)
if e.content_type == 'raw_ascii' and not getattr(settings, 'MAGELLAN_INDEX_RAW_ASCII', False):
if log:
print "Skipping page at url: %s, no means of extracting content" % processed_url
continue #don't index
title = e.get_title()
content = e.get_content()
headings = e.get_headings()
if log:
print "Adding page at url: %s, content length: %s to index" % (processed_url, len(content))
indexer.add_page(url=processed_url, title=title, content=content, site=profile.name, headings=headings)
# remove from the list of scheduled items
scheduled.remove(processed_url)
# store response status in the visited dictionary
visited[processed_url] = result_dict['response_status']
# enqueue any urls that need to be checked
if depth > 0:
for url in urls:
if url not in visited and url not in scheduled:
scheduled.add(url)
pending_urls.put((url, processed_url, depth - 1))
except KeyboardInterrupt:
pass
except Exception, e:
print "Got an exception while indexing page: %s\nWill exit." % processed_url
raise
