def wrapper(*args, **kwargs):
if settings.HAYSTACK_SEARCH_ENGINE == 'solr':
try:
soldadmin = SolrCoreAdmin(SearchQuerySet().query.backend.conn.url)
soldadmin.status()
func(*args, **kwargs)
except Exception, e:
logger.error('Error contacting SOLR backend: %s' %
getattr(e, 'reason', str(e)))
raise HaystackError('Error contacting SOLR backend.')
def check_solr_status(self, request):
""" Returns a boolean indicating if the Solr service is available. """
status = True
solr_admin = SolrCoreAdmin(settings.SOLR_ADMIN)
try:
solr_admin.status()
except ConnectionError:
# The Solr index is not accessible!
logger.error('Solr index unavailable', exc_info=True, extra={
'request': request, })
status = False
return status
class SolrCoreAdminTestCase(unittest.TestCase):
def assertIn(self, member, container, msg=None):
"""
Just like self.assertTrue(a in b), but with a nicer default message.
Backported from Python 2.7 unittest library.
"""
if member not in container:
standardMsg = '%s not found in %s' % (safe_repr(member),
safe_repr(container))
self.fail(self._formatMessage(msg, standardMsg))
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin('http://localhost:8080/solr/admin/cores')
def create(self, core_name):
return self.solr_admin.create(core_name, instance_dir=os.path.join(
os.path.dirname(__file__), 'fixtures'))
def test_status(self):
self.assertIn('<int name="status">0</int>', self.solr_admin.status())
self.assertIn('<int name="status">0</int>', self.solr_admin.status(core='core0'))
def test_create(self):
self.assertIn('<int name="status">0</int>', self.create('wheatley'))
def test_reload(self):
self.assertIn('<int name="status">0</int>', self.solr_admin.reload('wheatley'))
def test_rename(self):
self.create('wheatley')
self.assertIn('<int name="status">0</int>', self.solr_admin.rename('wheatley', 'rick'))
def test_swap(self):
self.create('wheatley')
self.create('rick')
self.assertIn('<int name="status">0</int>', self.solr_admin.swap('wheatley', 'rick'))
def test_unload(self):
self.create('wheatley')
self.assertIn('<int name="status">0</int>', self.solr_admin.unload('wheatley'))
def test_load(self):
self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')
class SolrCoreAdminTestCase(unittest.TestCase):
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin(
'http://localhost:8983/solr/admin/cores')
def test_status(self):
self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
self.assertTrue('<int name="status">' in self.solr_admin.status(
core='core0'))
def test_create(self):
self.assertTrue('<int name="status">0</int>' in create_core(
self.solr_admin, 'wheatley'))
def test_reload(self):
self.assertTrue(
'<int name="status">0</int>' in self.solr_admin.reload('wheatley'))
def test_rename(self):
self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename(
'wheatley', 'rick'))
def test_swap(self):
self.solr_admin.rename('rick', 'wheatley')
create_core(self.solr_admin, 'rick')
self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap(
'wheatley', 'rick'))
def test_unload(self):
self.assertTrue(
'<int name="status">0</int>' in self.solr_admin.unload('wheatley'))
def test_load(self):
self.assertRaises(NotImplementedError, self.solr_admin.load,
'wheatley')
def fin():
core_admin = SolrCoreAdmin("http://localhost:8989/solr/admin/cores")
core_admin.unload(solr_core_name)
target_dir = "test-solr/solr/{}".format(solr_core_name)
if os.path.isdir(target_dir):
shutil.rmtree(target_dir)
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin(
'http://localhost:8983/solr/admin/cores')
class SolrCoreAdminTestCase(unittest.TestCase):
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin("http://localhost:8983/solr/admin/cores")
def test_status(self):
self.assertIn('name="defaultCoreName"', self.solr_admin.status())
self.assertIn('<int name="status">', self.solr_admin.status(core="core0"))
def test_create(self):
self.assertIn('<int name="status">0</int>', self.solr_admin.create("wheatley"))
def test_reload(self):
self.assertIn('<int name="status">0</int>', self.solr_admin.reload("wheatley"))
def test_rename(self):
self.solr_admin.create("wheatley")
self.assertIn(
'<int name="status">0</int>', self.solr_admin.rename("wheatley", "rick")
)
def test_swap(self):
self.solr_admin.create("wheatley")
self.solr_admin.create("rick")
self.assertIn(
'<int name="status">0</int>', self.solr_admin.swap("wheatley", "rick")
)
def test_unload(self):
self.solr_admin.create("wheatley")
self.assertIn('<int name="status">0</int>', self.solr_admin.unload("wheatley"))
def test_load(self):
self.assertRaises(NotImplementedError, self.solr_admin.load, "wheatley")
def main():
options, remainder = getopt.gnu_getopt(sys.argv[1:], 'hmdtra', [
'help', 'migrate', 'dump', 'test', 'resume', 'async', 'solrhost=',
'eshost=', 'redishost=', 'index=', 'core=', 'solrfq=', 'solrid=',
'postgresqldsn=', 'translationmap=', 'esmapping=', 'essetting='
])
if len(sys.argv) == 1:
usage(sys.argv)
sys.exit()
aioloop = asyncio.get_event_loop()
with_asyncio = False
solrhost = 'solr'
solrfq = '*'
solrid = DEFAULT_ID_FIELD
eshost = 'elasticsearch'
redishost = 'redis'
postgresqldsn = None
core_name = 'solr2es'
index_name = None
action = 'migrate'
translationmap = None
esmapping = None
essetting = None
for opt, arg in options:
if opt in ('-h', '--help'):
usage(sys.argv)
sys.exit()
if opt in ('-a', '--async'):
with_asyncio = True
if opt == '--solrhost':
solrhost = arg
if opt == '--solrfq':
solrfq = arg
if opt == '--solrid':
solrid = arg
if opt == '--redishost':
redishost = arg
if opt == '--postgresqldsn':
postgresqldsn = arg
if opt == '--eshost':
eshost = arg
if opt == '--index':
index_name = arg
if opt == '--core':
core_name = arg
if opt == '--translationmap':
translationmap = TranslationMap(_get_dict_from_string_or_file(arg))
if opt == '--esmapping':
esmapping = _get_dict_from_string_or_file(arg)
if opt == '--essetting':
essetting = _get_dict_from_string_or_file(arg)
if opt in ('-d', '--dump'):
action = 'dump' if postgresqldsn is None else 'dump_pgsql'
elif opt in ('-r', '--resume'):
action = 'resume' if postgresqldsn is None else 'resume_pgsql'
elif opt in ('-m', '--migrate'):
action = 'migrate'
elif opt in ('-t', '--test'):
action = 'test'
if index_name is None:
index_name = core_name
solrurl = 'http://%s:8983/solr/%s' % (solrhost, core_name)
es_index_body = _get_es_mappings_and_settings(essetting, esmapping)
if action == 'migrate':
aioloop.run_until_complete(aiomigrate(solrurl, eshost, index_name, solrfq, solrid)) if with_asyncio \
else migrate(solrurl, eshost, index_name, solrfq, solrid)
elif action == 'dump':
aioloop.run_until_complete(aiodump_into_redis(solrurl, redishost, solrfq, solrid)) if with_asyncio \
else dump_into_redis(solrurl, redishost, solrfq, solrid)
elif action == 'resume':
aioloop.run_until_complete(
aioresume_from_redis(
redishost, eshost,
index_name)) if with_asyncio else resume_from_redis(
redishost, eshost, index_name)
elif action == 'dump_pgsql':
aioloop.run_until_complete(aiodump_into_pgsql(solrurl, postgresqldsn, solrfq, solrid)) if with_asyncio \
else dump_into_pgsql(solrurl, postgresqldsn, solrfq, solrid)
elif action == 'resume_pgsql':
aioloop.run_until_complete(
aioresume_from_pgsql(postgresqldsn, eshost, index_name, translationmap, es_index_body)) if with_asyncio \
else resume_from_postgresql(postgresqldsn, eshost, index_name, translationmap, es_index_body)
elif action == 'test':
solr_status = loads(
SolrCoreAdmin(
'http://%s:8983/solr/admin/cores?action=STATUS&core=%s' %
(solrhost, core_name)).status())
LOGGER.info('Elasticsearch ping on %s is %s', eshost,
'OK' if Elasticsearch(host=eshost).ping() else 'KO')
LOGGER.info('Solr status on %s is %s', solrurl,
'OK' if solr_status['status'][core_name] else 'KO')
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin('http://localhost:8983')
self.solr_admin.create('test_core', instance_dir="collection1")
class SolrCoreAdminTestCase(unittest.TestCase):
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin('http://localhost:8983')
self.solr_admin.create('test_core', instance_dir="collection1")
def tearDown(self):
try:
self.solr_admin.unload('test_core')
except:
# already cleaned up
pass
def test_status(self):
self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
self.assertTrue('<int name="status">' in self.solr_admin.status(core='test_core'))
def test_create(self):
result = self.solr_admin.create('wheatley', instance_dir="collection1")
self.assertTrue('<int name="status">0</int>' in result)
self.assertTrue('<str name="core">wheatley</str>' in result)
# cleanup
self.solr_admin.unload('wheatley')
def test_reload(self):
self.assertTrue('<int name="status">0</int>' in self.solr_admin.reload('test_core'))
def test_rename(self):
self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename('test_core', 'rick'))
res = self.solr_admin.status()
self.assertTrue('<lst name="rick">' in res)
self.assertFalse('<lst name="test_core">' in res)
def _get_uptime(self, core):
self.solr_admin.status(core)
uptime_pattern = r'<long name="uptime">(?P<uptime>\d+)</long>'
res = re.search(
uptime_pattern,
self.solr_admin.status(core)
).groupdict()
if res:
return int(res['uptime'])
else:
None
def test_swap(self):
self.solr_admin.create('rick', instance_dir="collection1")
#TODO find a better way to assert the swap than uptime
test_up = self._get_uptime('test_core')
rick_up = self._get_uptime('rick')
self.assertTrue(test_up > rick_up)
self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap('test_core', 'rick'))
res = self.solr_admin.status()
self.assertTrue('<lst name="rick">' in res)
self.assertTrue('<lst name="test_core">' in res)
test_up = self._get_uptime('test_core')
rick_up = self._get_uptime('rick')
self.assertTrue(test_up < rick_up)
# cleanup
self.solr_admin.unload('rick')
def test_unload(self):
self.solr_admin.create('wheatley', instance_dir="collection1")
self.assertTrue('<int name="status">0</int>' in self.solr_admin.unload('wheatley'))
def test_load(self):
self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')
def fin():
core_admin = SolrCoreAdmin('http://localhost:8989/solr/admin/cores')
core_admin.unload('phrase_match')
target_dir = 'test-solr/solr/{}'.format('phrase_match')
if os.path.isdir(target_dir):
shutil.rmtree(target_dir)
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin('http://localhost:8983/solr/admin/cores')
class SolrCoreAdminTestCase(unittest.TestCase):
def setUp(self):
super(SolrCoreAdminTestCase, self).setUp()
self.solr_admin = SolrCoreAdmin('http://localhost:8983/solr/admin/cores')
def test_status(self):
self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
self.assertTrue('<int name="status">' in self.solr_admin.status(core='core0'))
def test_create(self):
self.assertTrue('<int name="status">0</int>' in self.solr_admin.create('wheatley'))
def test_reload(self):
self.assertTrue('<int name="status">0</int>' in self.solr_admin.reload('wheatley'))
def test_rename(self):
self.solr_admin.create('wheatley')
self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename('wheatley', 'rick'))
def test_swap(self):
self.solr_admin.create('wheatley')
self.solr_admin.create('rick')
self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap('wheatley', 'rick'))
def test_unload(self):
self.solr_admin.create('wheatley')
self.assertTrue('<int name="status">0</int>' in self.solr_admin.unload('wheatley'))
def test_load(self):
self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')