def solr_add_ranges(id_ranges):
sub_range_length = task_get_option("flush")
id_ranges_to_index = []
for id_range in id_ranges:
lower_recid = id_range[0]
upper_recid = id_range[1]
i_low = lower_recid
while i_low <= upper_recid:
i_up = min(i_low + sub_range_length - 1, upper_recid)
id_ranges_to_index.append((i_low, i_up))
i_low += sub_range_length
tags_to_index = get_tags()
# Indexes latest records first by reversing
# This allows the ranker to return better results during long indexing
# runs as the ranker cuts the hitset using latest records
id_ranges_to_index.reverse()
next_commit_counter = 0
for id_range_to_index in id_ranges_to_index:
lower_recid = id_range_to_index[0]
upper_recid = id_range_to_index[1]
status_msg = "Solr ranking indexer called for %s-%s" % (lower_recid, upper_recid)
write_message(status_msg)
task_update_progress(status_msg)
next_commit_counter = solr_add_range(lower_recid, upper_recid, tags_to_index, next_commit_counter)
solr_commit_if_necessary(next_commit_counter, final_commit=True)
def solr_add_ranges(id_ranges):
sub_range_length = task_get_option("flush")
id_ranges_to_index = []
for id_range in id_ranges:
lower_recid = id_range[0]
upper_recid = id_range[1]
i_low = lower_recid
while i_low <= upper_recid:
i_up = min(i_low + sub_range_length - 1, upper_recid)
id_ranges_to_index.append((i_low, i_up))
i_low += sub_range_length
tags_to_index = get_tags()
# Indexes latest records first by reversing
# This allows the ranker to return better results during long indexing
# runs as the ranker cuts the hitset using latest records
id_ranges_to_index.reverse()
next_commit_counter = 0
for id_range_to_index in id_ranges_to_index:
lower_recid = id_range_to_index[0]
upper_recid = id_range_to_index[1]
status_msg = "Solr ranking indexer called for %s-%s" % (lower_recid,
upper_recid)
write_message(status_msg)
task_update_progress(status_msg)
next_commit_counter = solr_add_range(lower_recid, upper_recid,
tags_to_index,
next_commit_counter)
solr_commit_if_necessary(next_commit_counter, final_commit=True)
def test_load_tags(self):
"""solrutils - load tags"""
self.assertEqual(
{
'abstract': ['520__%'],
'author': ['100__a', '700__a'],
'title': ['245__%', '246__%'],
'keyword': ['6531_a']
}, get_tags())
def test_build_default_field_content(self):
"""solrutils - build default field content"""
tags = get_tags()
self.assertEqual(u'Ellis, J Enqvist, K Nanopoulos, D V',
get_field_content_in_utf8(18, 'author', tags))
self.assertEqual(u'Kahler manifolds gravitinos axions constraints noscale',
get_field_content_in_utf8(18, 'keyword', tags))
self.assertEqual(u'In 1962, CERN hosted the 11th International Conference on High Energy Physics. Among the distinguished visitors were eight Nobel prizewinners.Left to right: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang and Robert Hofstadter.',
get_field_content_in_utf8(6, 'abstract', tags))
def test_build_default_field_content(self):
"""solrutils - build default field content"""
tags = get_tags()
self.assertEqual(u'Ellis, J Enqvist, K Nanopoulos, D V',
get_field_content_in_utf8(18, 'author', tags))
self.assertEqual(
u'Kahler manifolds gravitinos axions constraints noscale',
get_field_content_in_utf8(18, 'keyword', tags))
self.assertEqual(
u'In 1962, CERN hosted the 11th International Conference on High Energy Physics. Among the distinguished visitors were eight Nobel prizewinners.Left to right: Cecil F. Powell, Isidor I. Rabi, Werner Heisenberg, Edwin M. McMillan, Emile Segre, Tsung Dao Lee, Chen Ning Yang and Robert Hofstadter.',
get_field_content_in_utf8(6, 'abstract', tags))
def test_load_tags(self):
"""solrutils - load tags"""
self.assertEqual({'abstract': ['520__%'], 'author': ['100__a', '700__a'], 'title': ['245__%', '246__%'], 'keyword': ['6531_a']},
get_tags())
