def test_search(self):
"""Tests searching against more than one document
"""
# index directory
idxr = indexer.get_indexer(name='test', rebuild_index=True)
path = os.path.join(self.test_dir, 'text')
idxr.index_text(path)
self.assertTrue(
idxr.doc_count() == 7,
"Bad document index count, expected 7 but, indexed %d" %
idxr.doc_count())
# search
idx = idxr.get_index()
search_text = 'value'
results = idx.search(search_text)
self.assertTrue(
len(results) > 1,
'not enough results from the search, expected more than 1, but found %d'
% len(results))
# search by path
results = idx.search_path(os.path.join(path, 'objc_example.m'))
self.assertTrue(
len(results) == 1,
'wrong number of results for the path search, expected 1, but found %d'
% len(results))
def test_index_directory(self):
"""Tests directory content indexing logic
"""
path = os.path.join(self.test_dir, 'text')
idx = indexer.get_indexer(name='test', rebuild_index=True)
idx.index_text(path)
# test values
self.assertTrue(idx.doc_count() == 7, 'bad doc count, expected 7 but, indexed %d' % idx.doc_count())
def test_index_directory(self):
"""Tests directory content indexing logic
"""
path = os.path.join(self.test_dir, 'text')
idx = indexer.get_indexer(name='test', rebuild_index=True)
idx.index_text(path)
# test values
self.assertTrue(idx.doc_count() == 7, 'bad doc count, expected 7 but, indexed %d' % idx.doc_count())
def test_index_file(self):
"""Tests file indexing logic
"""
path = os.path.join(self.test_dir, 'text/objc_example.m')
idx = indexer.get_indexer(name='test', rebuild_index=True)
idx.index_text(path)
# test values
self.assertTrue(idx.doc_count() == 1, 'bad doc count')
def test_index_file(self):
"""Tests file indexing logic
"""
path = os.path.join(self.test_dir, 'text/objc_example.m')
idx = indexer.get_indexer(name='test', rebuild_index=True)
idx.index_text(path)
# test values
self.assertTrue(idx.doc_count() == 1, 'bad doc count')
def test_indexer_creation(self):
"""Test indexer creation logic
"""
# the below call provides and sets up a default indexer environment
# based on the settings.py script values
# the index name (usually 'main') has been overriden for testing
idx = indexer.get_indexer(name='test', rebuild_index=True)
# test values
self.assertFalse(idx is None, 'unable to create an indexer')
def test_indexer_creation(self):
"""Test indexer creation logic
"""
# the below call provides and sets up a default indexer environment
# based on the settings.py script values
# the index name (usually 'main') has been overriden for testing
idx = indexer.get_indexer(name='test', rebuild_index=True)
# test values
self.assertFalse(idx is None, 'unable to create an indexer')
def show_stats():
# backend stats
print 'Available indexer backends: %s' % backends.indexer_names()
print 'Available searcher backends: %s' % backends.searcher_names()
print 'Current backend: %s' % settings.DEFAULT_SEARCHER
# indexer stats
idxr = indexer.get_indexer()
print 'Total documents indexed: %d' % idxr.doc_count()
# database stats
print 'Index Database: %s' % db.DATABASE_PATH
def show_stats():
# backend stats
print 'Available indexer backends: %s' % backends.indexer_names()
print 'Available searcher backends: %s' % backends.searcher_names()
print 'Current backend: %s' % settings.DEFAULT_SEARCHER
# indexer stats
idxr = indexer.get_indexer(rebuild_index=False)
print 'Total documents indexed: %d' % idxr.doc_count()
# database stats
print 'Index Database: %s' % db.DATABASE_PATH
def show_stats():
# backend stats
print "Available indexer backends: %s" % backends.indexer_names()
print "Available searcher backends: %s" % backends.searcher_names()
print "Current backend: %s" % settings.DEFAULT_SEARCHER
# indexer stats
idxr = indexer.get_indexer(rebuild_index=False)
print "Total documents indexed: %d" % idxr.doc_count()
# database stats
print "Index Database: %s" % db.DATABASE_PATH
def test_suggestions(self):
"""Test suggestion logic"""
idxr = indexer.get_indexer(name='test', rebuild_index=True)
path = os.path.join(self.test_dir, 'text')
idxr.index_text(path)
# get suggestion
idx = idxr.get_index()
search_text = 'var'
result = idx.suggestions(search_text)
self.assertTrue(result, 'no suggestions returned')
self.assertIn('val', result, 'suggestion not matching')
self.assertNotIn(search_text, result, 'original query should not be included')
def test_suggestions(self):
"""Test suggestion logic"""
idxr = indexer.get_indexer(name='test', rebuild_index=True)
path = os.path.join(self.test_dir, 'text')
idxr.index_text(path)
# get suggestion
idx = idxr.get_index()
search_text = 'var'
result = idx.suggestions(search_text)
self.assertTrue(result, 'no suggestions returned')
self.assertIn('val', result, 'suggestion not matching')
self.assertNotIn(search_text, result,
'original query should not be included')
def results_from_search_text(text, pagenum=1, isPath=False, type=None):
"""Returns the results from the search using the given text, populated with the transformed items
"""
idx = indexer.get_indexer(writable=False).get_index()
# find something in the index
if isPath:
results = idx.search_path(text)
else:
try:
results = idx.search(text, pagenum, core_settings.RESULTS_PER_PAGE)
except ValueError, e:
# This assumes the value error resulted from an page count issue
app.logger.error('Out of page bounds: %s' % e)
return []
def test_simple_search(self):
"""Tests simple search logic
"""
# index a file for the search
path = os.path.join(self.test_dir, 'text/objc_example.m')
idxr = indexer.get_indexer(name='test', rebuild_index=True)
idxr.index_text(path)
# test values
self.assertTrue(idxr.doc_count() == 1, 'bad doc count, expected 1 but, found %d' % idxr.doc_count())
idx = idxr.get_index()
# find something in the file
results = idx.search('key')
self.assertTrue(len(results) == 1, 'wrong hit count, expected 1 but, found %d' % len(results))
def results_from_search_text(text, pagenum=1, isPath=False, type=None):
"""Returns the results from the search using the given text, populated with the transformed items
"""
idx = indexer.get_indexer(writable=False).get_index()
# find something in the index
if isPath:
results = idx.search_path(text)
else:
try:
results = idx.search(text, pagenum, core_settings.RESULTS_PER_PAGE)
except ValueError, e:
# This assumes the value error resulted from an page count issue
app.logger.error('Out of page bounds: %s' % e)
return []
def run():
options = get_options()
# determine app action
if options.run_tests:
tests.run_all()
elif options.show_version:
pyver = sys.version_info
print ' Python: v%d.%d.%d' % (pyver.major, pyver.minor, pyver.micro)
print 'Sherlock: v' + get_version_info('sherlock')
print ' Flask: v' + get_version_info('flask')
print 'Pygments: v' + get_version_info('pygments')
print ' Whoosh: v' + get_version_info('whoosh')
print 'CherryPy: v' + get_version_info('cherrypy')
elif options.show_stats:
# backend stats
print 'Available indexer backends: %s' % backends.indexer_names()
print 'Available searcher backends: %s' % backends.searcher_names()
print 'Current backend: %s' % settings.DEFAULT_SEARCHER
# indexer stats
idxr = indexer.get_indexer()
print 'Total documents indexed: %d' % idxr.doc_count()
# database stats
print 'Index Database: %s' % db.DATABASE_PATH
elif options.run_server:
print 'Backend: %s' % settings.DEFAULT_SEARCHER
print 'Server: %s' % server.get_server_type()
# launch web server
server.run()
elif options.reindex:
path = utils.resolve_path(settings.INDEX_PATH)
# check path
if not path.endswith('/'):
raise Exception('INDEX_PATH must end with a trailing slash. %s' %
path)
if not os.path.exists(path):
raise Exception('Check INDEX_PATH. Does it exist? %s' % path)
print 'Indexing path: %s' % path
if FORCE_INDEX_REBUILD:
wait_time = 5 # seconds to wait/pause until rebuilding index
print 'Reindexing everything!'
print 'Waiting %ss for interrupt...' % wait_time
import time
time.sleep(wait_time)
indexer.index_path(path)
# record indexed time
SherlockMeta.set('last_indexed',
datetime.now().strftime(SHORT_DATE_FORMAT))
else:
print 'Use -h to see options.'
pass
def run():
options = get_options()
# determine app action
if options.run_tests:
tests.run_all()
elif options.show_version:
pyver = sys.version_info
print ' Python: v%d.%d.%d' % (pyver.major, pyver.minor, pyver.micro)
print 'Sherlock: v' + get_version_info('sherlock')
print ' Flask: v' + get_version_info('flask')
print 'Pygments: v' + get_version_info('pygments')
print ' Whoosh: v' + get_version_info('whoosh')
print 'CherryPy: v' + get_version_info('cherrypy')
elif options.show_stats:
# backend stats
print 'Available indexer backends: %s' % backends.indexer_names()
print 'Available searcher backends: %s' % backends.searcher_names()
print 'Current backend: %s' % settings.DEFAULT_SEARCHER
# indexer stats
idxr = indexer.get_indexer()
print 'Total documents indexed: %d' % idxr.doc_count()
# database stats
print 'Index Database: %s' % db.DATABASE_PATH
elif options.run_server:
print 'Backend: %s' % settings.DEFAULT_SEARCHER
print 'Server: %s' % server.get_server_type()
# launch web server
server.run()
elif options.reindex:
path = utils.resolve_path(settings.INDEX_PATH)
# check path
if not path.endswith('/'):
raise Exception('INDEX_PATH must end with a trailing slash. %s' % path)
if not os.path.exists(path):
raise Exception('Check INDEX_PATH. Does it exist? %s' % path)
print 'Indexing path: %s' % path
if FORCE_INDEX_REBUILD:
wait_time = 5 # seconds to wait/pause until rebuilding index
print 'Reindexing everything!'
print 'Waiting %ss for interrupt...' % wait_time
import time
time.sleep(wait_time)
indexer.index_path(path)
# record indexed time
SherlockMeta.set('last_indexed', datetime.now().strftime(SHORT_DATE_FORMAT))
else:
print 'Use -h to see options.'
pass
def test_search(self):
"""Tests searching against more than one document
"""
# index directory
idxr = indexer.get_indexer(name='test', rebuild_index=True)
path = os.path.join(self.test_dir, 'text')
idxr.index_text(path)
self.assertTrue(idxr.doc_count() == 7, "Bad document index count, expected 7 but, indexed %d" % idxr.doc_count())
# search
idx = idxr.get_index()
search_text = 'value'
results = idx.search(search_text)
self.assertTrue(len(results) > 1, 'not enough results from the search, expected more than 1, but found %d' % len(results))
# search by path
results = idx.search_path(os.path.join(path, 'objc_example.m'))
self.assertTrue(len(results) == 1, 'wrong number of results for the path search, expected 1, but found %d' % len(results))
def test_simple_unicode_search(self):
"""Tests simple search logic using a unicode string
"""
# index a file for the search
path = os.path.join(self.test_dir, 'text/example.py')
idxr = indexer.get_indexer(name='test', rebuild_index=True)
idxr.index_text(path)
# test values
self.assertTrue(
idxr.doc_count() == 1,
'bad doc count, expected 1 but, found %d' % idxr.doc_count())
idx = idxr.get_index()
# find some unicode in the file
results = idx.search('©opyright')
self.assertTrue(
len(results) == 1,
'wrong hit count, expected 1 but, found %d' % len(results))
def _test_html_transform(self):
"""Tests the HTML transform operation
"""
# index a file for the search
path = os.path.join(self.test_dir, 'text/objc_example.m')
idxr = indexer.get_indexer(name='test', rebuild_index=True)
idxr.index_text(path)
idx = idxr.get_index()
# find something in the file
results = idx.search('nsstring')
self.assertTrue(len(results) == 1, 'bad results count')
# transform the results
trns = transformer.Transformer()
items = trns.transform_results(results)
self.assertTrue(len(items) == 1, 'no transformed items')
html = items[0].html
# debug()
self.assertTrue(len(html) > 0, 'no HTML returned')
def _test_html_transform(self):
"""Tests the HTML transform operation
"""
# index a file for the search
path = os.path.join(self.test_dir, "text/objc_example.m")
idxr = indexer.get_indexer(name="test", rebuild_index=True)
idxr.index_text(path)
idx = idxr.get_index()
# find something in the file
results = idx.search("nsstring")
self.assertTrue(len(results) == 1, "bad results count")
result = results[0]
# transform the results
trns = transformer.Transformer()
items = trns.transform_results(results)
self.assertTrue(len(items) == 1, "no transformed items")
html = items[0].html
# debug()
self.assertTrue(len(html) > 0, "no HTML returned")
def suggestion_from_search_text(text):
idx = indexer.get_indexer(writable=False).get_index()
return idx.suggestions(text)
