def handle(self, *directories, **options):
from django.db import transaction
self.style = color_style()
verbose = options.get('verbose')
force = options.get('force')
show_traceback = options.get('traceback', False)
import_picture = options.get('import_picture')
wait_until = None
if options.get('wait_until'):
wait_until = time.mktime(time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
if verbose > 0:
print "Will wait until %s; it's %f seconds from now" % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(wait_until)), wait_until - time.time())
index = None
if options.get('search_index') and not settings.NO_SEARCH_INDEX:
index = Index()
try:
index.index_tags()
index.index.commit()
except Exception, e:
index.index.rollback()
raise e
def handle(self, *directories, **options):
from django.db import transaction
self.style = color_style()
verbose = options.get('verbose')
force = options.get('force')
show_traceback = options.get('traceback', False)
import_picture = options.get('import_picture')
wait_until = None
if options.get('wait_until'):
wait_until = time.mktime(
time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
if verbose > 0:
print "Will wait until %s; it's %f seconds from now" % (
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(wait_until)), wait_until - time.time())
index = None
if options.get('search_index') and not settings.NO_SEARCH_INDEX:
index = Index()
try:
index.index_tags()
index.index.commit()
except Exception, e:
index.index.rollback()
raise e
def setUp(self):
WLTestCase.setUp(self)
index = Index()
self.search = Search()
index.delete_query(self.search.index.query(uid="*"))
index.index.commit()
with self.settings(NO_SEARCH_INDEX=False):
self.do_doktora = Book.from_xml_file(
get_fixture('do-doktora.xml', opds))
self.do_anusie = Book.from_xml_file(
get_fixture('fraszka-do-anusie.xml', catalogue))
def setUp(self):
WLTestCase.setUp(self)
index = Index()
self.search = Search()
index.delete_query(self.search.index.query(uid="*"))
index.index.commit()
with self.settings(NO_SEARCH_INDEX=False):
self.do_doktora = Book.from_xml_file(
get_fixture('do-doktora.xml', opds))
self.do_anusie = Book.from_xml_file(
get_fixture('fraszka-do-anusie.xml', catalogue))
def handle(self, *args, **opts):
index = Index()
index.open()
self.delete_old(index)
try:
index.optimize()
finally:
index.close()
def setUp(self):
JVM.attachCurrentThread()
WLTestCase.setUp(self)
settings.NO_SEARCH_INDEX = False
settings.SEARCH_INDEX = path.join(settings.MEDIA_ROOT, 'search')
txt = path.join(path.dirname(__file__), 'files/fraszka-do-anusie.xml')
self.book = models.Book.from_xml_file(txt)
index = Index()
index.open()
try:
index.index_book(self.book)
except:
index.close()
self.search = Search()
def setUp(self):
"""
Setup index that will be subjected to the tests.
"""
self.index = Index(sample_stop_words())
class IndexTests(unittest.TestCase):
"""
Test case for Index class.
"""
def setUp(self):
"""
Setup index that will be subjected to the tests.
"""
self.index = Index(sample_stop_words())
def test_sample_indexing_with_no_exceptions(self):
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
self.index.build_index([sample1, sample2, sample3])
def test_invalid_term_search(self):
"""
Test if the search returns when the term is not found.
"""
sample1 = Indexable(1, "this is an indexable simple metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["not_valid_term"])
self.assertItemsEqual(search_results, expected_indices)
def test_mixed_valid_invalid_term_search(self):
"""
Test if the search returns when there are valid and invalid terms mixed.
"""
sample1 = Indexable(1, "this is an indexable simple metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable metadata")
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["not_valid_term", "super"])
self.assertItemsEqual(search_results, expected_indices)
def test_one_term_search(self):
"""
Test if the search for one term returns expected results.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = [1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["super"])
self.assertItemsEqual(search_results, expected_indices)
def test_stop_word_search(self):
"""
Test if stop words are correctly ignored.
"""
sample1 = Indexable(1, "this is an indexable metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["this"])
self.assertItemsEqual(search_results, expected_indices)
def test_two_terms_search(self):
"""
Test if the search for two term returns expected results.
"""
sample1 = Indexable(1, "this is an indexable simple metadata")
sample2 = Indexable(2, "this is an indexable super metadata")
sample3 = Indexable(3, "this is another indexable super metadata")
expected_indices = [1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(["indexable", "super"])
self.assertItemsEqual(search_results, expected_indices)
def handle(self, *directories, **options):
from django.db import transaction
self.style = color_style()
verbose = options.get('verbose')
force = options.get('force')
show_traceback = options.get('traceback', False)
import_picture = options.get('import_picture')
wait_until = None
if options.get('wait_until'):
wait_until = time.mktime(time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
if verbose > 0:
print "Will wait until %s; it's %f seconds from now" % (
time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(wait_until)), wait_until - time.time())
if options.get('search_index') and not settings.NO_SEARCH_INDEX:
index = Index()
index.open()
try:
index.index_tags()
finally:
index.close()
# Start transaction management.
transaction.commit_unless_managed()
transaction.enter_transaction_management()
transaction.managed(True)
files_imported = 0
files_skipped = 0
for dir_name in directories:
if not os.path.isdir(dir_name):
print self.style.ERROR("%s: Not a directory. Skipping." % dir_name)
else:
# files queue
files = sorted(os.listdir(dir_name))
postponed = {}
while files:
file_name = files.pop(0)
file_path = os.path.join(dir_name, file_name)
file_base, ext = os.path.splitext(file_path)
# Skip files that are not XML files
if not ext == '.xml':
continue
if verbose > 0:
print "Parsing '%s'" % file_path
else:
sys.stdout.write('.')
sys.stdout.flush()
# Import book files
try:
if import_picture:
self.import_picture(file_path, options)
else:
self.import_book(file_path, options)
files_imported += 1
transaction.commit()
except (Book.AlreadyExists, Picture.AlreadyExists):
print self.style.ERROR('%s: Book or Picture already imported. Skipping. To overwrite use --force.' %
file_path)
files_skipped += 1
except Book.DoesNotExist, e:
if file_name not in postponed or postponed[file_name] < files_imported:
# push it back into the queue, maybe the missing child will show up
if verbose:
print self.style.NOTICE('Waiting for missing children')
files.append(file_name)
postponed[file_name] = files_imported
else:
# we're in a loop, nothing's being imported - some child is really missing
raise e
def setUp(self):
"""
Setup index that will be subjected to the tests.
"""
self.index = Index(sample_stop_words())
class IndexTests(unittest.TestCase):
"""
Test case for Index class.
"""
def setUp(self):
"""
Setup index that will be subjected to the tests.
"""
self.index = Index(sample_stop_words())
def test_sample_indexing_with_no_exceptions(self):
sample1 = Indexable(1, 'this is an indexable metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable metadata')
self.index.build_index([sample1, sample2, sample3])
def test_invalid_term_search(self):
"""
Test if the search returns when the term is not found.
"""
sample1 = Indexable(1, 'this is an indexable simple metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable metadata')
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(['not_valid_term'])
self.assertItemsEqual(search_results, expected_indices)
def test_mixed_valid_invalid_term_search(self):
"""
Test if the search returns when there are valid and invalid terms mixed.
"""
sample1 = Indexable(1, 'this is an indexable simple metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable metadata')
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(['not_valid_term', 'super'])
self.assertItemsEqual(search_results, expected_indices)
def test_one_term_search(self):
"""
Test if the search for one term returns expected results.
"""
sample1 = Indexable(1, 'this is an indexable metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable super metadata')
expected_indices = [1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(['super'])
self.assertItemsEqual(search_results, expected_indices)
def test_stop_word_search(self):
"""
Test if stop words are correctly ignored.
"""
sample1 = Indexable(1, 'this is an indexable metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable super metadata')
expected_indices = []
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(['this'])
self.assertItemsEqual(search_results, expected_indices)
def test_two_terms_search(self):
"""
Test if the search for two term returns expected results.
"""
sample1 = Indexable(1, 'this is an indexable simple metadata')
sample2 = Indexable(2, 'this is an indexable super metadata')
sample3 = Indexable(3, 'this is another indexable super metadata')
expected_indices = [1, 2]
self.index.build_index([sample1, sample2, sample3])
search_results = self.index.search_terms(['indexable', 'super'])
self.assertItemsEqual(search_results, expected_indices)
}, {
'selector': '3',
'prompt': 'Exit',
'return': 'exit'
}]
selection = prompt.options("Browser or Terminal Engine", options)
if selection == 'exit':
print 'Bye'
exit()
global my_index, dirname
dirname = path = prompt.query('Documents Path',
default=os.getcwd(),
validators=[validators.PathValidator()])
my_index = Index(dirname)
if selection == 'web':
app.run(debug=False)
elif selection == 'cmd':
while 1:
options = [{
'selector': '1',
'prompt': 'Query',
'return': 'cont'
}, {
'selector': '2',
'prompt': 'Exit',
'return': 'exit'
}]
