Пример #1
0
 def wrapper(*args, **kwargs):
     if settings.HAYSTACK_SEARCH_ENGINE == 'solr':
         try:
             soldadmin = SolrCoreAdmin(SearchQuerySet().query.backend.conn.url)
             soldadmin.status()
             func(*args, **kwargs)
         except Exception, e:
             logger.error('Error contacting SOLR backend: %s' % 
                 getattr(e, 'reason', str(e)))
             raise HaystackError('Error contacting SOLR backend.')
Пример #2
0
 def check_solr_status(self, request):
     """ Returns a boolean indicating if the Solr service is available. """
     status = True
     solr_admin = SolrCoreAdmin(settings.SOLR_ADMIN)
     try:
         solr_admin.status()
     except ConnectionError:
         # The Solr index is not accessible!
         logger.error('Solr index unavailable', exc_info=True, extra={
             'request': request, })
         status = False
     return status
Пример #3
0
class SolrCoreAdminTestCase(unittest.TestCase):
    def assertIn(self, member, container, msg=None):
        """
        Just like self.assertTrue(a in b), but with a nicer default message.
        Backported from Python 2.7 unittest library.
        """
        if member not in container:
            standardMsg = '%s not found in %s' % (safe_repr(member),
                                                  safe_repr(container))
            self.fail(self._formatMessage(msg, standardMsg))


    def setUp(self):
        super(SolrCoreAdminTestCase, self).setUp()
        self.solr_admin = SolrCoreAdmin('http://localhost:8080/solr/admin/cores')

    def create(self, core_name):
        return self.solr_admin.create(core_name, instance_dir=os.path.join(
            os.path.dirname(__file__), 'fixtures'))

    def test_status(self):
        self.assertIn('<int name="status">0</int>', self.solr_admin.status())
        self.assertIn('<int name="status">0</int>', self.solr_admin.status(core='core0'))

    def test_create(self):
        self.assertIn('<int name="status">0</int>', self.create('wheatley'))

    def test_reload(self):
        self.assertIn('<int name="status">0</int>', self.solr_admin.reload('wheatley'))

    def test_rename(self):
        self.create('wheatley')
        self.assertIn('<int name="status">0</int>', self.solr_admin.rename('wheatley', 'rick'))

    def test_swap(self):
        self.create('wheatley')
        self.create('rick')
        self.assertIn('<int name="status">0</int>', self.solr_admin.swap('wheatley', 'rick'))

    def test_unload(self):
        self.create('wheatley')
        self.assertIn('<int name="status">0</int>', self.solr_admin.unload('wheatley'))

    def test_load(self):
        self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')
Пример #4
0
class SolrCoreAdminTestCase(unittest.TestCase):
    def setUp(self):
        super(SolrCoreAdminTestCase, self).setUp()
        self.solr_admin = SolrCoreAdmin(
            'http://localhost:8983/solr/admin/cores')

    def test_status(self):
        self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
        self.assertTrue('<int name="status">' in self.solr_admin.status(
            core='core0'))

    def test_create(self):
        self.assertTrue('<int name="status">0</int>' in create_core(
            self.solr_admin, 'wheatley'))

    def test_reload(self):
        self.assertTrue(
            '<int name="status">0</int>' in self.solr_admin.reload('wheatley'))

    def test_rename(self):
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename(
            'wheatley', 'rick'))

    def test_swap(self):
        self.solr_admin.rename('rick', 'wheatley')
        create_core(self.solr_admin, 'rick')
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap(
            'wheatley', 'rick'))

    def test_unload(self):
        self.assertTrue(
            '<int name="status">0</int>' in self.solr_admin.unload('wheatley'))

    def test_load(self):
        self.assertRaises(NotImplementedError, self.solr_admin.load,
                          'wheatley')
Пример #5
0
 def fin():
     core_admin = SolrCoreAdmin("http://localhost:8989/solr/admin/cores")
     core_admin.unload(solr_core_name)
     target_dir = "test-solr/solr/{}".format(solr_core_name)
     if os.path.isdir(target_dir):
         shutil.rmtree(target_dir)
Пример #6
0
 def setUp(self):
     super(SolrCoreAdminTestCase, self).setUp()
     self.solr_admin = SolrCoreAdmin(
         'http://localhost:8983/solr/admin/cores')
Пример #7
0
class SolrCoreAdminTestCase(unittest.TestCase):
    def setUp(self):
        super(SolrCoreAdminTestCase, self).setUp()
        self.solr_admin = SolrCoreAdmin("http://localhost:8983/solr/admin/cores")

    def test_status(self):
        self.assertIn('name="defaultCoreName"', self.solr_admin.status())
        self.assertIn('<int name="status">', self.solr_admin.status(core="core0"))

    def test_create(self):
        self.assertIn('<int name="status">0</int>', self.solr_admin.create("wheatley"))

    def test_reload(self):
        self.assertIn('<int name="status">0</int>', self.solr_admin.reload("wheatley"))

    def test_rename(self):
        self.solr_admin.create("wheatley")
        self.assertIn(
            '<int name="status">0</int>', self.solr_admin.rename("wheatley", "rick")
        )

    def test_swap(self):
        self.solr_admin.create("wheatley")
        self.solr_admin.create("rick")
        self.assertIn(
            '<int name="status">0</int>', self.solr_admin.swap("wheatley", "rick")
        )

    def test_unload(self):
        self.solr_admin.create("wheatley")
        self.assertIn('<int name="status">0</int>', self.solr_admin.unload("wheatley"))

    def test_load(self):
        self.assertRaises(NotImplementedError, self.solr_admin.load, "wheatley")
Пример #8
0
def main():
    options, remainder = getopt.gnu_getopt(sys.argv[1:], 'hmdtra', [
        'help', 'migrate', 'dump', 'test', 'resume', 'async', 'solrhost=',
        'eshost=', 'redishost=', 'index=', 'core=', 'solrfq=', 'solrid=',
        'postgresqldsn=', 'translationmap=', 'esmapping=', 'essetting='
    ])
    if len(sys.argv) == 1:
        usage(sys.argv)
        sys.exit()

    aioloop = asyncio.get_event_loop()
    with_asyncio = False
    solrhost = 'solr'
    solrfq = '*'
    solrid = DEFAULT_ID_FIELD
    eshost = 'elasticsearch'
    redishost = 'redis'
    postgresqldsn = None
    core_name = 'solr2es'
    index_name = None
    action = 'migrate'
    translationmap = None
    esmapping = None
    essetting = None
    for opt, arg in options:
        if opt in ('-h', '--help'):
            usage(sys.argv)
            sys.exit()

        if opt in ('-a', '--async'):
            with_asyncio = True

        if opt == '--solrhost':
            solrhost = arg

        if opt == '--solrfq':
            solrfq = arg

        if opt == '--solrid':
            solrid = arg

        if opt == '--redishost':
            redishost = arg

        if opt == '--postgresqldsn':
            postgresqldsn = arg

        if opt == '--eshost':
            eshost = arg

        if opt == '--index':
            index_name = arg

        if opt == '--core':
            core_name = arg

        if opt == '--translationmap':
            translationmap = TranslationMap(_get_dict_from_string_or_file(arg))

        if opt == '--esmapping':
            esmapping = _get_dict_from_string_or_file(arg)

        if opt == '--essetting':
            essetting = _get_dict_from_string_or_file(arg)

        if opt in ('-d', '--dump'):
            action = 'dump' if postgresqldsn is None else 'dump_pgsql'
        elif opt in ('-r', '--resume'):
            action = 'resume' if postgresqldsn is None else 'resume_pgsql'
        elif opt in ('-m', '--migrate'):
            action = 'migrate'
        elif opt in ('-t', '--test'):
            action = 'test'

    if index_name is None:
        index_name = core_name

    solrurl = 'http://%s:8983/solr/%s' % (solrhost, core_name)
    es_index_body = _get_es_mappings_and_settings(essetting, esmapping)

    if action == 'migrate':
        aioloop.run_until_complete(aiomigrate(solrurl, eshost, index_name, solrfq, solrid)) if with_asyncio \
            else migrate(solrurl, eshost, index_name, solrfq, solrid)
    elif action == 'dump':
        aioloop.run_until_complete(aiodump_into_redis(solrurl, redishost, solrfq, solrid)) if with_asyncio \
            else dump_into_redis(solrurl, redishost, solrfq, solrid)
    elif action == 'resume':
        aioloop.run_until_complete(
            aioresume_from_redis(
                redishost, eshost,
                index_name)) if with_asyncio else resume_from_redis(
                    redishost, eshost, index_name)
    elif action == 'dump_pgsql':
        aioloop.run_until_complete(aiodump_into_pgsql(solrurl, postgresqldsn, solrfq, solrid)) if with_asyncio \
            else dump_into_pgsql(solrurl, postgresqldsn, solrfq, solrid)
    elif action == 'resume_pgsql':
        aioloop.run_until_complete(
            aioresume_from_pgsql(postgresqldsn, eshost, index_name, translationmap, es_index_body)) if with_asyncio \
            else resume_from_postgresql(postgresqldsn, eshost, index_name, translationmap, es_index_body)
    elif action == 'test':
        solr_status = loads(
            SolrCoreAdmin(
                'http://%s:8983/solr/admin/cores?action=STATUS&core=%s' %
                (solrhost, core_name)).status())
        LOGGER.info('Elasticsearch ping on %s is %s', eshost,
                    'OK' if Elasticsearch(host=eshost).ping() else 'KO')
        LOGGER.info('Solr status on %s is %s', solrurl,
                    'OK' if solr_status['status'][core_name] else 'KO')
Пример #9
0
 def setUp(self):
     super(SolrCoreAdminTestCase, self).setUp()
     self.solr_admin = SolrCoreAdmin('http://localhost:8983')
     self.solr_admin.create('test_core', instance_dir="collection1")
Пример #10
0
class SolrCoreAdminTestCase(unittest.TestCase):
    def setUp(self):
        super(SolrCoreAdminTestCase, self).setUp()
        self.solr_admin = SolrCoreAdmin('http://localhost:8983')
        self.solr_admin.create('test_core', instance_dir="collection1")

    def tearDown(self):
        try:
            self.solr_admin.unload('test_core')
        except:
            # already cleaned up
            pass

    def test_status(self):
        self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
        self.assertTrue('<int name="status">' in self.solr_admin.status(core='test_core'))

    def test_create(self):
        result = self.solr_admin.create('wheatley', instance_dir="collection1")
        self.assertTrue('<int name="status">0</int>' in result)
        self.assertTrue('<str name="core">wheatley</str>' in result)
        # cleanup
        self.solr_admin.unload('wheatley')

    def test_reload(self):
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.reload('test_core'))

    def test_rename(self):
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename('test_core', 'rick'))
        res = self.solr_admin.status()
        self.assertTrue('<lst name="rick">' in res)
        self.assertFalse('<lst name="test_core">' in res)

    def _get_uptime(self, core):
        self.solr_admin.status(core)
        uptime_pattern = r'<long name="uptime">(?P<uptime>\d+)</long>'
        res = re.search(
            uptime_pattern,
            self.solr_admin.status(core)
        ).groupdict()
        if res:
            return int(res['uptime'])
        else:
            None
    
    def test_swap(self):
        self.solr_admin.create('rick', instance_dir="collection1")
        #TODO find a better way to assert the swap than uptime
        test_up = self._get_uptime('test_core')
        rick_up = self._get_uptime('rick')
        self.assertTrue(test_up > rick_up)
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap('test_core', 'rick'))
        res = self.solr_admin.status()
        self.assertTrue('<lst name="rick">' in res)
        self.assertTrue('<lst name="test_core">' in res)
        test_up = self._get_uptime('test_core')
        rick_up = self._get_uptime('rick')
        self.assertTrue(test_up < rick_up)

        # cleanup
        self.solr_admin.unload('rick')

    def test_unload(self):
        self.solr_admin.create('wheatley', instance_dir="collection1")
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.unload('wheatley'))

    def test_load(self):
        self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')
Пример #11
0
 def fin():
     core_admin = SolrCoreAdmin('http://localhost:8989/solr/admin/cores')
     core_admin.unload('phrase_match')
     target_dir = 'test-solr/solr/{}'.format('phrase_match')
     if os.path.isdir(target_dir):
         shutil.rmtree(target_dir)
Пример #12
0
 def setUp(self):
     super(SolrCoreAdminTestCase, self).setUp()
     self.solr_admin = SolrCoreAdmin('http://localhost:8983/solr/admin/cores')
Пример #13
0
class SolrCoreAdminTestCase(unittest.TestCase):
    def setUp(self):
        super(SolrCoreAdminTestCase, self).setUp()
        self.solr_admin = SolrCoreAdmin('http://localhost:8983/solr/admin/cores')

    def test_status(self):
        self.assertTrue('name="defaultCoreName"' in self.solr_admin.status())
        self.assertTrue('<int name="status">' in self.solr_admin.status(core='core0'))

    def test_create(self):
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.create('wheatley'))

    def test_reload(self):
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.reload('wheatley'))

    def test_rename(self):
        self.solr_admin.create('wheatley')
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.rename('wheatley', 'rick'))

    def test_swap(self):
        self.solr_admin.create('wheatley')
        self.solr_admin.create('rick')
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.swap('wheatley', 'rick'))

    def test_unload(self):
        self.solr_admin.create('wheatley')
        self.assertTrue('<int name="status">0</int>' in self.solr_admin.unload('wheatley'))

    def test_load(self):
        self.assertRaises(NotImplementedError, self.solr_admin.load, 'wheatley')