def test_postings_file_seek():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
pfile.seek(10)
assert_eq(10, pfile.pointer)
os.remove(filename)
def test_runner_ascii(patched_func):
cmd = ['cat', os.path.join(TEST_DIR, 'ascii_text.txt')]
for line in run_command(cmd):
if type(line) == int:
assert_eq(line, 0)
else:
assert_eq(line.rstrip(), 'Hello ASCII!')
def test_postings_file_get_entry_reset_false():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
last = 12
for i in xrange(1, last):
current_entry = PostingsFileEntry(i)
current_entry.own_pointer = pfile.pointer
pfile.write_entry(current_entry)
if i != last - 1:
current_entry.next_pointer = pfile.pointer
pfile.write_entry(current_entry)
entries = []
entry = pfile.get_entry(head, reset=False)
while entry:
entries.append(entry)
entry = entry.next()
entries = [entry.doc_id for entry in entries]
assert_eq([1,2,3,4,5,6,7,8,9,10,11], entries)
os.remove(filename)
def test_dot_product_generators():
vals = [0,1,2,3,4]
v1 = (x for x in vals)
v2 = (x for x in vals)
assert_eq(
1 + 4 + 9 + 16,
dot_product(v1, v2))
def test_postings_file_get_entry_reset_false():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
last = 12
for i in xrange(1, last):
current_entry = PostingsFileEntry(i)
current_entry.own_pointer = pfile.pointer
pfile.write_entry(current_entry)
if i != last - 1:
current_entry.next_pointer = pfile.pointer
pfile.write_entry(current_entry)
entries = []
entry = pfile.get_entry(head, reset=False)
while entry:
entries.append(entry)
entry = entry.next()
entries = [entry.doc_id for entry in entries]
assert_eq([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], entries)
os.remove(filename)
def test_unit_vector_generator():
vals = [0, 1, 2, 3, 4]
vector = (x for x in vals)
length = math.sqrt(1 + 4 + 9 + 16)
assert_eq((float(0) / length, float(1) / length,
float(2) / length, float(3) / length, float(4) / length),
tuple(unit_vector(vector)))
def test_runner_ascii(patched_func):
cmd = ['cat', os.path.join(TEST_DIR,'ascii_text.txt')]
for line in run_command(cmd):
if type(line) == int:
assert_eq(line, 0)
else:
assert_eq(line.rstrip(), 'Hello ASCII!')
def test_postings_file_seek():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
pfile.seek(10)
assert_eq(10, pfile.pointer)
os.remove(filename)
def test_runner_unicode(patched_func):
cmd = ['cat', os.path.join(TEST_DIR, 'unicode_text.txt')]
for line in run_command(cmd):
if type(line) == int:
assert_eq(line, 0)
else:
if line.rstrip() != 'Hello Unicode\u203d':
print('WARNING: Unicode reading not supported!')
def test_runner_unicode(patched_func):
cmd = ['cat', os.path.join(TEST_DIR,'unicode_text.txt')]
for line in run_command(cmd):
if type(line) == int:
assert_eq(line, 0)
else:
if line.rstrip() != 'Hello Unicode\u203d':
print('WARNING: Unicode reading not supported!')
def test_unit_vector_generator():
vals = [0,1,2,3,4]
vector = (x for x in vals)
length = math.sqrt(1 + 4 + 9 + 16)
assert_eq(
(float(0)/length, float(1)/length, float(2)/length, float(3)/length,
float(4)/length),
tuple(unit_vector(vector)))
def test_unit_vector_multiple():
vector = (1, 2)
length = math.sqrt(1**2 + 2**2)
assert_eq((float(1) / length, float(2) / length),
tuple(unit_vector(vector)))
vector = (3, 5)
length = math.sqrt(3**2 + 5**2)
assert_eq((float(3) / length, float(5) / length),
tuple(unit_vector(vector)))
def test_ordered_dict_get_item():
o = OrderedDict()
o['a'] = 1
o['b'] = 'asdf'
assert_eq(1, o['a'])
assert_eq(1, o.get('a'))
assert_eq('asdf', o['b'])
assert_eq('asdf', o.get('b'))
assert_eq(None, o.get('c'))
def test_cache_box_is_cached():
def test_func(x):
return x
cached_test_func = CacheBox(test_func)
assert not cached_test_func.is_cached(5)
assert_eq(5, cached_test_func(5))
assert cached_test_func.is_cached(5)
assert not cached_test_func.is_cached(4)
def test_unit_vector_multiple():
vector = (1, 2)
length = math.sqrt(1**2 + 2**2)
assert_eq(
(float(1)/length, float(2)/length),
tuple(unit_vector(vector)))
vector = (3, 5)
length = math.sqrt(3**2 + 5**2)
assert_eq(
(float(3)/length, float(5)/length),
tuple(unit_vector(vector)))
def test_cache_decorators():
@cached_function()
def foo():
return 1
@cached_function(cache_size=10)
def bar():
return 1
assert_eq('foo', foo.__name__)
assert_eq('bar', bar.__name__)
def test_ordered_dict_set_item():
o = OrderedDict()
o['a'] = 1
o['b'] = 'asdf'
assert_eq(1, o['a'])
assert_eq('asdf', o['b'])
try:
o['c']
assert False, 'Should throw key error'
except KeyError:
pass
def test_postings_file_get_entry():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
pfile.write_entry(PostingsFileEntry(1))
# Test that we set the entries own pointer.
assert_eq(head, pfile.get_entry(head).own_pointer)
ptr = pfile.pointer
pfile.write_entry(PostingsFileEntry(2))
assert_eq(ptr, pfile.get_entry(ptr).own_pointer)
os.remove(filename)
def test_postings_file_write_entry_overwrite():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
write_location = 0
entry = PostingsFileEntry(1)
entry.own_pointer = write_location
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1).to_string(), pfile.read_entry(write_location))
entry.next_pointer = 2
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1, 2).to_string(),
pfile.read_entry(write_location))
entry.skip_pointer = 3
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1, 2, 3).to_string(),
pfile.read_entry(write_location))
os.remove(filename)
def test_postings_file_get_entry():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
pfile.write_entry(PostingsFileEntry(1))
# Test that we set the entries own pointer.
assert_eq(head, pfile.get_entry(head).own_pointer)
ptr = pfile.pointer
pfile.write_entry(PostingsFileEntry(2))
assert_eq(ptr, pfile.get_entry(ptr).own_pointer)
os.remove(filename)
def test_postings_file_write_entry_overwrite():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
write_location = 0
entry = PostingsFileEntry(1)
entry.own_pointer = write_location
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1).to_string(),
pfile.read_entry(write_location))
entry.next_pointer = 2
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1, 2).to_string(),
pfile.read_entry(write_location))
entry.skip_pointer = 3
pfile.write_entry(entry)
assert_eq(
PostingsFileEntry(1, 2, 3).to_string(),
pfile.read_entry(write_location))
os.remove(filename)
def test_infix_to_prefix_complex():
query = 'a AND b OR c'
assert_eq(['OR', 'AND', 'a', 'b', 'c'], infix_to_prefix(query))
query = 'a AND NOT b'
assert_eq(['AND', 'a', 'NOT', 'b'], infix_to_prefix(query))
query = 'a AND NOT b OR c'
assert_eq(['OR', 'AND', 'a', 'NOT', 'b', 'c'], infix_to_prefix(query))
query = 'a AND (NOT b) OR c'
assert_eq(['OR', 'AND', 'a', 'NOT', 'b', 'c'], infix_to_prefix(query))
query = 'a AND (b OR NOT c) AND d'
assert_eq(['AND', 'a', 'AND', 'OR', 'b', 'NOT', 'c', 'd'], infix_to_prefix(query))
def test_infix_to_prefix_complex():
query = 'a AND b OR c'
assert_eq(['OR', 'AND', 'a', 'b', 'c'], infix_to_prefix(query))
query = 'a AND NOT b'
assert_eq(['AND', 'a', 'NOT', 'b'], infix_to_prefix(query))
query = 'a AND NOT b OR c'
assert_eq(['OR', 'AND', 'a', 'NOT', 'b', 'c'], infix_to_prefix(query))
query = 'a AND (NOT b) OR c'
assert_eq(['OR', 'AND', 'a', 'NOT', 'b', 'c'], infix_to_prefix(query))
query = 'a AND (b OR NOT c) AND d'
assert_eq(['AND', 'a', 'AND', 'OR', 'b', 'NOT', 'c', 'd'],
infix_to_prefix(query))
def test_postings_file_get_entry_from_pointer():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
pfile.write_entry(1)
for i in xrange(10):
next_ptr = pfile.pointer
pfile.write_entry(i + 1, next_ptr, write_location=prev_ptr)
pfile.write_entry(i + 2, write_location=next_ptr)
prev_ptr = next_ptr
entries = pfile.get_entry_list_from_pointer(head)
entries = [entry.doc_id for entry in entries]
assert_eq([1,2,3,4,5,6,7,8,9,10,11], entries)
os.remove(filename)
def test_postings_file_get_entry_from_pointer():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
pfile.write_entry(1)
for i in xrange(10):
next_ptr = pfile.pointer
pfile.write_entry(i + 1, next_ptr, write_location=prev_ptr)
pfile.write_entry(i + 2, write_location=next_ptr)
prev_ptr = next_ptr
entries = pfile.get_entry_list_from_pointer(head)
entries = [entry.doc_id for entry in entries]
assert_eq([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], entries)
os.remove(filename)
def test_ordered_dict_popitem():
o = OrderedDict()
o['a'] = 1
o['b'] = 'asdf'
assert_eq(1, o.popitem(True))
assert_eq(1, len(o))
o['a'] = 1
assert_eq(1, o.popitem())
assert_eq(1, len(o))
def test_list_node():
ln = ListNode.from_list([])
assert ln is None
ln = ListNode.from_list([1, 2, 3])
assert_eq(1, ln.val())
ln = ln.next()
assert_eq(2, ln.val())
ln = ln.next()
assert_eq(3, ln.val())
ln = ln.next()
assert_eq(None, ln)
def test_list_node():
ln = ListNode.from_list([])
assert ln is None
ln = ListNode.from_list([1, 2, 3])
assert_eq(1, ln.val())
ln = ln.next()
assert_eq(2, ln.val())
ln = ln.next()
assert_eq(3, ln.val())
ln = ln.next()
assert_eq(None, ln)
def test_dictionary_all_terms():
d = Dictionary()
assert_eq([], d.all_terms())
d.add_term('asdf', 1, 1)
assert_eq(['asdf'], d.all_terms())
d.add_term('asdf', 2, 1)
assert_eq(['asdf'], d.all_terms())
d.add_term('qwer', 1, 1)
d.add_term('zxcv', 1, 1)
assert_eq(sorted(['asdf', 'qwer', 'zxcv']), sorted(d.all_terms()))
def test_dictionary_all_docs():
d = Dictionary()
assert_eq([], d.all_docs())
d.add_term('asdf', 1, 1)
assert_eq([1], d.all_docs())
d.add_term('asdf', 2, 1)
assert_eq([1, 2], d.all_docs())
d.add_term('qwer', 1, 1)
d.add_term('zxcv', 1, 1)
assert_eq([1, 2], d.all_docs())
def test_dictionary_all_docs():
d = Dictionary()
assert_eq([], d.all_docs())
d.add_term('asdf', 1, 1)
assert_eq([1], d.all_docs())
d.add_term('asdf', 2, 1)
assert_eq([1, 2], d.all_docs())
d.add_term('qwer', 1, 1)
d.add_term('zxcv', 1, 1)
assert_eq([1, 2], d.all_docs())
def test_postings_file_get_entry_from_pointer():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
last = 12
for i in xrange(1, last):
current_entry = PostingsFileEntry(i)
current_entry.own_pointer = pfile.pointer
pfile.write_entry(current_entry)
if i != last - 1:
current_entry.next_pointer = pfile.pointer
pfile.write_entry(current_entry)
entries = pfile.get_entry_list_from_pointer(head)
entries = [entry.doc_id for entry in entries]
assert_eq([1,2,3,4,5,6,7,8,9,10,11], entries)
os.remove(filename)
def test_postings_file_get_entry_from_pointer():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
head = pfile.pointer
prev_ptr = head
last = 12
for i in xrange(1, last):
current_entry = PostingsFileEntry(i)
current_entry.own_pointer = pfile.pointer
pfile.write_entry(current_entry)
if i != last - 1:
current_entry.next_pointer = pfile.pointer
pfile.write_entry(current_entry)
entries = pfile.get_entry_list_from_pointer(head)
entries = [entry.doc_id for entry in entries]
assert_eq([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], entries)
os.remove(filename)
def test_dictionary_all_terms():
d = Dictionary()
assert_eq([], d.all_terms())
d.add_term('asdf', 1, 1)
assert_eq(['asdf'], d.all_terms())
d.add_term('asdf', 2, 1)
assert_eq(['asdf'], d.all_terms())
d.add_term('qwer', 1, 1)
d.add_term('zxcv', 1, 1)
assert_eq(
sorted(['asdf', 'qwer', 'zxcv']),
sorted(d.all_terms()))
def test_unit_vector_single():
vector = (1, )
assert_eq((1, ), tuple(unit_vector(vector)))
vector = (2, )
assert_eq((1, ), tuple(unit_vector(vector)))
vector = (10, )
assert_eq((1, ), tuple(unit_vector(vector)))
def test_infix_to_prefix_single_operator():
query = 'a AND b'
assert_eq(['AND', 'a', 'b'], infix_to_prefix(query))
query = 'a OR b'
assert_eq(['OR', 'a', 'b'], infix_to_prefix(query))
query = 'NOT b'
assert_eq(['NOT', 'b'], infix_to_prefix(query))
def test_unit_vector_single():
vector = (1,)
assert_eq((1,), tuple(unit_vector(vector)))
vector = (2,)
assert_eq((1,), tuple(unit_vector(vector)))
vector = (10,)
assert_eq((1,), tuple(unit_vector(vector)))
def test_infix_to_prefix_single_operator():
query = 'a AND b'
assert_eq(['AND', 'a', 'b'], infix_to_prefix(query))
query = 'a OR b'
assert_eq(['OR', 'a', 'b'], infix_to_prefix(query))
query = 'NOT b'
assert_eq(['NOT', 'b'], infix_to_prefix(query))
def test_postings_file_entry_eq():
pfe1 = PostingsFileEntry(1)
pfe2 = PostingsFileEntry(1, 10)
pfe3 = PostingsFileEntry(1, 10, 100)
pfe4 = PostingsFileEntry(1, 10, 100, 1000)
assert_eq(pfe1, pfe1)
assert_eq(pfe2, pfe2)
assert_eq(pfe3, pfe3)
assert_eq(pfe4, pfe4)
assert not pfe1 == pfe2
assert not pfe1 == pfe3
assert not pfe1 == pfe4
assert not pfe2 == pfe3
assert not pfe2 == pfe4
assert not pfe3 == pfe4
def test_postings_file_entry_eq():
pfe1 = PostingsFileEntry(1)
pfe2 = PostingsFileEntry(1, 10)
pfe3 = PostingsFileEntry(1, 10, 100)
pfe4 = PostingsFileEntry(1, 10, 100, 1000)
assert_eq(pfe1, pfe1)
assert_eq(pfe2, pfe2)
assert_eq(pfe3, pfe3)
assert_eq(pfe4, pfe4)
assert not pfe1 == pfe2
assert not pfe1 == pfe3
assert not pfe1 == pfe4
assert not pfe2 == pfe3
assert not pfe2 == pfe4
assert not pfe3 == pfe4
def test_lru_cache_box():
def test_func(x):
return x
lru_cached_test_func = LRUCacheBox(test_func, cache_size=10)
assert_eq(0, lru_cached_test_func.current_size())
lru_cached_test_func(1)
assert_eq(1, lru_cached_test_func.current_size())
lru_cached_test_func.invalidate(1)
assert_eq(0, lru_cached_test_func.current_size())
for i in xrange(50):
lru_cached_test_func(i)
assert_eq(10, lru_cached_test_func.current_size())
def test_dot_product():
v1 = (1, )
v2 = (2, )
assert_eq(1 * 2, dot_product(v1, v2))
v1 = (1, 1)
v2 = (2, 2)
assert_eq(1 * 2 + 1 * 2, dot_product(v1, v2))
v1 = (3, 6)
v2 = (2, 5)
assert_eq(3 * 2 + 6 * 5, dot_product(v1, v2))
def test_postings_file_write_entry():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
pfile.write_entry(PostingsFileEntry(1))
pfile.write_entry(PostingsFileEntry(2))
pfile.write_entry(PostingsFileEntry(3))
assert_eq(PostingsFileEntry(1).to_string(), pfile.read_entry(0))
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(0 + PostingsFileEntry.SIZE))
assert_eq(
PostingsFileEntry(3).to_string(),
pfile.read_entry(0 + PostingsFileEntry.SIZE * 2))
os.remove(filename)
def test_postings_file_write_entry():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
pfile.write_entry(PostingsFileEntry(1))
pfile.write_entry(PostingsFileEntry(2))
pfile.write_entry(PostingsFileEntry(3))
assert_eq(
PostingsFileEntry(1).to_string(),
pfile.read_entry(0))
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(0 + PostingsFileEntry.SIZE))
assert_eq(
PostingsFileEntry(3).to_string(),
pfile.read_entry(0 + PostingsFileEntry.SIZE * 2))
os.remove(filename)
def test_naive_class_method_cache():
x = {}
x['val'] = 0
class Test(object):
@naive_class_method_cache
def blah(self):
x['val'] += 1
t = Test()
assert_eq(0, x['val'])
t.blah()
assert_eq(1, x['val'])
t.blah()
assert_eq(1, x['val'])
# Invalidate
t.blah.invalidate()
t.blah()
assert_eq(2, x['val'])
def test_postings_file_write_entry_out_of_order():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
first_write_location = pfile.pointer
first_entry = PostingsFileEntry(1)
first_entry.own_pointer = first_write_location
pfile.write_entry(first_entry)
assert_eq(
PostingsFileEntry(1).to_string(),
pfile.read_entry(first_write_location))
second_write_location = pfile.pointer
second_entry = PostingsFileEntry(2)
second_entry.own_pointer = second_write_location
pfile.write_entry(second_entry)
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(second_write_location))
# Update first entry
first_entry.doc_id = 4
pfile.write_entry(first_entry)
assert_eq(
first_entry.to_string(),
pfile.read_entry(first_write_location))
# Add third entry
third_entry = PostingsFileEntry(3)
pfile.write_entry(third_entry)
# Check that second write location was not overwritten.
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(second_write_location))
os.remove(filename)
def test_postings_file_write_entry_out_of_order():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
first_write_location = pfile.pointer
first_entry = PostingsFileEntry(1)
first_entry.own_pointer = first_write_location
pfile.write_entry(first_entry)
assert_eq(
PostingsFileEntry(1).to_string(),
pfile.read_entry(first_write_location))
second_write_location = pfile.pointer
second_entry = PostingsFileEntry(2)
second_entry.own_pointer = second_write_location
pfile.write_entry(second_entry)
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(second_write_location))
# Update first entry
first_entry.doc_id = 4
pfile.write_entry(first_entry)
assert_eq(first_entry.to_string(),
pfile.read_entry(first_write_location))
# Add third entry
third_entry = PostingsFileEntry(3)
pfile.write_entry(third_entry)
# Check that second write location was not overwritten.
assert_eq(
PostingsFileEntry(2).to_string(),
pfile.read_entry(second_write_location))
os.remove(filename)
def test_ordered_dict_del_item():
o = OrderedDict()
o['a'] = 1
o['b'] = 'asdf'
assert_eq(1, o['a'])
assert_eq('asdf', o['b'])
del o['a']
assert_eq(None, o.get('a'))
try:
o['a']
assert False, 'Should throw key error'
except KeyError:
pass
def test_postings_file_write_entry_overwrite():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
write_location = 0
pfile.write_entry(1, write_location=write_location)
assert_eq(
PostingsFileEntry(1).to_string(), pfile.read_entry(write_location))
pfile.write_entry(1, 2, write_location=write_location)
assert_eq(
PostingsFileEntry(1, 2).to_string(),
pfile.read_entry(write_location))
pfile.write_entry(1, 2, 3, write_location=write_location)
assert_eq(
PostingsFileEntry(1, 2, 3).to_string(),
pfile.read_entry(write_location))
os.remove(filename)
def test_dot_product():
v1 = (1,)
v2 = (2,)
assert_eq(1 * 2, dot_product(v1, v2))
v1 = (1, 1)
v2 = (2, 2)
assert_eq(
1 * 2 + 1 * 2,
dot_product(v1, v2))
v1 = (3, 6)
v2 = (2, 5)
assert_eq(
3 * 2 + 6 * 5,
dot_product(v1, v2))
def test_postings_file_write_entry_overwrite():
filename = 'test'
with PostingsFile(filename, 'w+') as pfile:
assert_eq(0, pfile.pointer)
write_location = 0
pfile.write_entry(1, write_location=write_location)
assert_eq(
PostingsFileEntry(1).to_string(),
pfile.read_entry(write_location))
pfile.write_entry(1, 2, write_location=write_location)
assert_eq(
PostingsFileEntry(1, 2).to_string(),
pfile.read_entry(write_location))
pfile.write_entry(1, 2, 3, write_location=write_location)
assert_eq(
PostingsFileEntry(1, 2, 3).to_string(),
pfile.read_entry(write_location))
os.remove(filename)
def test_ordered_dict_len():
o = OrderedDict()
o['a'] = 1
o['b'] = 'asdf'
assert_eq(1, o['a'])
assert_eq('asdf', o['b'])
assert_eq(2, len(o))
del o['a']
assert_eq(1, len(o))
o['c'] = 'qwer'
assert_eq(2, len(o))
del o['c']
del o['b']
assert_eq(0, len(o))
def test_cache_box_simple():
x = {}
x['val'] = 0
def test_func():
x['val'] += 1
return x['val']
cached_test_func = CacheBox(test_func)
assert_eq(0, x['val'])
assert_eq(1, cached_test_func())
# Value does not change.
assert_eq(1, cached_test_func())
assert_eq(1, x['val'])
assert_eq(1, cached_test_func())
assert_eq(1, x['val'])
