def test_request_request__omitHeder_is_false(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path', {
"fake_params": "fake_value",
'omitHeader': 'false'
},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'false',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
assert response == {'fake_data': 'fake_value'}
def test_request_request__malformed_response(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = "Malformed Response"
mock_request.return_value = fake_response
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Parsing Error: Malformed Response")
def test_request_request__server_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return ["http://localsolr:7070/solr/"]
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_any_call(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
self.assertEqual(client.current_hosts,
["http://localsolr:7070/solr/"])
mock_request.assert_any_call(
'GET',
'http://localsolr:7070/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
def test_request_get(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
client.get("fake_path", params={"fake_param": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'GET')
def test_request_no_active_pool(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return []
mock_zookeeper.side_effect = get_active_hosts
client = SolrRequest([
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
],
zookeeper_hosts=[
"http://localzook:2181",
"http://localzook:2181"
])
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return [
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
]
mock_zookeeper.side_effect = get_active_hosts
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
def test_request_request__all_servers_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return []
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
self.assertEqual(client.current_hosts, [])
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
with self.assertRaises(SolrError) as cm:
response2 = client.request(
'fake_path', {"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
def test_request_refill_pool(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
# simulate an empty pool and a 5 minute old error
client.current_hosts = ["http://localsolr:8080/solr/"]
client.last_error = time.time() - 5
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
self.assertEqual(client.current_hosts, client.master_hosts)
def test_request_no_active_pool(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
client.current_hosts = ''
# simulate an empty pool
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
# add nodes back to the bool
client.current_hosts = [
"http://localsolr:7070/solr/", "http://localsolr:8080/solr/"
]
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
def test_should_refresh(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_empty_active_hosts():
return []
def get_active_hosts():
return [
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
]
mock_zookeeper.side_effect = get_empty_active_hosts
client = SolrRequest([
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
],
zookeeper_hosts=[
"http://localzook:2181",
"http://localzook:2181"
])
mock_zookeeper.side_effect = get_active_hosts
assert client.master_hosts == []
client._last_request = time.time() - (
client.refresh_frequency * 1000 + 3000)
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"},
headers={'foo': 'bar'})
assert client.master_hosts == get_active_hosts()
assert client.current_hosts == client.master_hosts
class TestSolrZookRequest(unittest.TestCase):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
def test_request_post(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
client.post("fake_path",
params={"fake_param": "fake_value"},
body={"fake_data": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'POST',
body={"fake_data": "fake_value"})
def test_request_get(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
client.get("fake_path", params={"fake_param": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'GET')
def test_request_request__server_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return ["http://localsolr:7070/solr/"]
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_any_call(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
self.assertEqual(client.current_hosts,
["http://localsolr:7070/solr/"])
mock_request.assert_any_call(
'GET',
'http://localsolr:7070/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error), "Server down!")
def test_request_request__all_servers_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return []
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
self.assertEqual(client.current_hosts, [])
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
with self.assertRaises(SolrError) as cm:
response2 = client.request(
'fake_path', {"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
def test_request_refill_pool(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
# simulate an empty pool and a 5 minute old error
client.current_hosts = ["http://localsolr:8080/solr/"]
client.last_error = time.time() - 5
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
self.assertEqual(client.current_hosts, client.master_hosts)
def test_request_no_active_pool(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
client.current_hosts = ''
# simulate an empty pool
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"SOLR reporting all nodes as down")
# add nodes back to the bool
client.current_hosts = [
"http://localsolr:7070/solr/", "http://localsolr:8080/solr/"
]
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
class TestSolrZookRequest(unittest.TestCase):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
def test_request_post(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
client.post("fake_path",
params={"fake_param": "fake_value"},
body={"fake_data": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'POST',
body={"fake_data": "fake_value"},
headers=None)
def test_request_get(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
client.get("fake_path", params={"fake_param": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'GET',
headers=None)
def test_request_request__server_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return ["http://localsolr:7070/solr/"]
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_any_call(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
self.assertEqual(client.current_hosts,
["http://localsolr:7070/solr/"])
mock_request.assert_any_call(
'GET',
'http://localsolr:7070/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
def test_request_request__all_servers_down(self):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"],
zookeeper_hosts=["http://localzook:2181", "http://localzook:2181"])
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return []
mock_zookeeper.side_effect = get_active_hosts
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
self.assertEqual(client.current_hosts, [])
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
with self.assertRaises(SolrError) as cm:
response2 = client.request(
'fake_path', {"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
def test_request_no_active_pool(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return []
mock_zookeeper.side_effect = get_active_hosts
client = SolrRequest([
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
],
zookeeper_hosts=[
"http://localzook:2181",
"http://localzook:2181"
])
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_active_hosts():
return [
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
]
mock_zookeeper.side_effect = get_active_hosts
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
def test_should_refresh(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
with mock.patch('wukong.zookeeper.Zookeeper.get_active_hosts'
) as mock_zookeeper:
def get_empty_active_hosts():
return []
def get_active_hosts():
return [
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
]
mock_zookeeper.side_effect = get_empty_active_hosts
client = SolrRequest([
"http://localsolr:7070/solr/",
"http://localsolr:8080/solr/"
],
zookeeper_hosts=[
"http://localzook:2181",
"http://localzook:2181"
])
mock_zookeeper.side_effect = get_active_hosts
assert client.master_hosts == []
client._last_request = time.time() - (
client.refresh_frequency * 1000 + 3000)
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"},
headers={'foo': 'bar'})
assert client.master_hosts == get_active_hosts()
assert client.current_hosts == client.master_hosts
class TestSolrRequest(unittest.TestCase):
client = SolrRequest(
["http://localsolr:7070/solr/", "http://localsolr:8080/solr/"])
def test_request_post(self):
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
self.client.post("fake_path",
params={"fake_param": "fake_value"},
body={"fake_data": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'POST',
body={"fake_data": "fake_value"},
headers=None)
def test_request_get(self):
with mock.patch('wukong.request.SolrRequest.request') as mock_request:
self.client.get("fake_path", params={"fake_param": "fake_value"})
mock_request.assert_called_once_with('fake_path',
{"fake_param": "fake_value"},
'GET',
headers=None)
def test_request_request(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
assert response == {'fake_data': 'fake_value'}
def test_request_request__omitHeder_is_false(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path', {
"fake_params": "fake_value",
'omitHeader': 'false'
},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'false',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
assert response == {'fake_data': 'fake_value'}
def test_request_request__empty_params(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = json.dumps({'fake_data': 'fake_value'})
mock_request.return_value = fake_response
response = client.request('fake_path',
None,
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
def test_request_request__status_code(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 400
fake_response.reason = "Test Error"
mock_request.return_value = fake_response
with self.assertRaises(SolrError) as cm:
response = self.client.request(
'fake_path', {"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_any_call(
'GET',
'http://localsolr:8080/solr/fake_path',
data={"fake_body": "fake_value"},
headers={'content-type': 'application/json'},
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
timeout=15)
mock_request.assert_any_call(
'GET',
'http://localsolr:7070/solr/fake_path',
data={"fake_body": "fake_value"},
headers={'content-type': 'application/json'},
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
def test_request_request__malformed_response(self):
client = SolrRequest(["http://localsolr:8080/solr/"])
with mock.patch('requests.sessions.Session.request') as mock_request:
fake_response = Response()
fake_response.status_code = 200
fake_response.text = "Malformed Response"
mock_request.return_value = fake_response
with self.assertRaises(SolrError) as cm:
response = client.request('fake_path',
{"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_called_once_with(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Parsing Error: Malformed Response")
def test_request_request__server_down(self):
with mock.patch('requests.sessions.Session.request') as mock_request:
def request(*args, **kwargs):
raise ConnectionError("Server down!")
mock_request.side_effect = request
with self.assertRaises(SolrError) as cm:
response = self.client.request(
'fake_path', {"fake_params": "fake_value"},
'GET',
body={"fake_body": "fake_value"})
mock_request.assert_any_call(
'GET',
'http://localsolr:8080/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
mock_request.assert_any_call(
'GET',
'http://localsolr:7070/solr/fake_path',
params={
"fake_params": "fake_value",
'wt': 'json',
'omitHeader': 'true',
'json.nl': 'map'
},
headers={'content-type': 'application/json'},
data={"fake_body": "fake_value"},
timeout=15)
solr_error = cm.exception
self.assertEqual(str(solr_error),
"Unable to fetch from any SOLR nodes")
def __init__(self,
solr_hosts,
solr_collection,
zookeeper_hosts=None,
timeout=15,
zookeeper_timeout=5):
"""
Do all the interactions with SOLR server
(e.g. update, select, get and delete)
:param solr_hosts: the hosts for SOLR.
:type server: str
:param solr_collection: the name of the collection in SOLR.
:type solr_collection: str
:param zookeeper_hosts: the hosts for zookeeper.
:type zookeeper_hosts: str
:param timeout: the timeout for request to SOLR.
:type timeout: int
"""
if solr_hosts is None and zookeeper_hosts is not None:
logger.info('Getting solr hosts from zookeeper for collection %s',
solr_collection)
zk = Zookeeper(zookeeper_hosts, zookeeper_timeout)
solr_hosts = zk.get_active_hosts(collection_name=solr_collection)
if solr_hosts is None or solr_collection is None:
logger.error('Neither solr_hosts nor solr_collection has been set')
raise solr_errors.SolrError(
"Either solr_hosts or solr_collection can not be None")
if not isinstance(solr_hosts, list):
solr_hosts = solr_hosts.split(",")
if zookeeper_hosts is not None:
hostnames, sep, chroot = zookeeper_hosts.rpartition('/')
# If hostnames is empty then there is no chroot. Set it to empty.
if not hostnames:
chroot = ''
else:
chroot = '/%s' % chroot
logger.debug('Using solr via zookeeper at chroot %s', chroot)
self.zookeeper_hosts = [
"http://%s%s" % (
host,
chroot,
) for host in zookeeper_hosts.split(",")
]
logger.info('Connected to zookeeper hosts at %s',
self.zookeeper_hosts)
else:
logger.debug('Not using zookeeper for SolrCloud')
self.zookeeper_hosts = None
logger.info('Connected to solr hosts %s', solr_hosts)
self.solr_hosts = [_format_solr_url(host) for host in solr_hosts]
self.solr_collection = solr_collection
self.client = SolrRequest(solr_hosts=self.solr_hosts,
zookeeper_hosts=zookeeper_hosts,
timeout=timeout)
class SolrAPI(object):
def __init__(self,
solr_hosts,
solr_collection,
zookeeper_hosts=None,
timeout=15,
zookeeper_timeout=5):
"""
Do all the interactions with SOLR server
(e.g. update, select, get and delete)
:param solr_hosts: the hosts for SOLR.
:type server: str
:param solr_collection: the name of the collection in SOLR.
:type solr_collection: str
:param zookeeper_hosts: the hosts for zookeeper.
:type zookeeper_hosts: str
:param timeout: the timeout for request to SOLR.
:type timeout: int
"""
if solr_hosts is None and zookeeper_hosts is not None:
logger.info('Getting solr hosts from zookeeper for collection %s',
solr_collection)
zk = Zookeeper(zookeeper_hosts, zookeeper_timeout)
solr_hosts = zk.get_active_hosts(collection_name=solr_collection)
if solr_hosts is None or solr_collection is None:
logger.error('Neither solr_hosts nor solr_collection has been set')
raise solr_errors.SolrError(
"Either solr_hosts or solr_collection can not be None")
if not isinstance(solr_hosts, list):
solr_hosts = solr_hosts.split(",")
if zookeeper_hosts is not None:
hostnames, sep, chroot = zookeeper_hosts.rpartition('/')
# If hostnames is empty then there is no chroot. Set it to empty.
if not hostnames:
chroot = ''
else:
chroot = '/%s' % chroot
logger.debug('Using solr via zookeeper at chroot %s', chroot)
self.zookeeper_hosts = [
"http://%s%s" % (
host,
chroot,
) for host in zookeeper_hosts.split(",")
]
logger.info('Connected to zookeeper hosts at %s',
self.zookeeper_hosts)
else:
logger.debug('Not using zookeeper for SolrCloud')
self.zookeeper_hosts = None
logger.info('Connected to solr hosts %s', solr_hosts)
self.solr_hosts = [_format_solr_url(host) for host in solr_hosts]
self.solr_collection = solr_collection
self.client = SolrRequest(solr_hosts=self.solr_hosts,
zookeeper_hosts=zookeeper_hosts,
timeout=timeout)
def _get_collection_url(self, path):
return "%s/%s" % (self.solr_collection, path)
def is_alive(self):
"""
Check if current collection is live from zookeeper.
:return: weather or not if the collection is live
:rtype: boolean
"""
params = {'detail': 'true', 'path': '/clusterstate.json'}
try:
response = self.client.get('zookeeper', params)
except solr_errors.SolrError:
logger.exception('Failed to check zookeeper')
return False
else:
try:
data = json.loads(response['znode']['data'])
except ValueError:
return False
for name, collection in data.items():
shards = collection['shards']
for shard, shard_info in shards.items():
replicas = shard_info['replicas']
for replica, info in replicas.items():
state = info['state']
if name == self.solr_collection and state != 'active':
return False
return True
def update(self, docs, commit=False):
"""
Add new docs or updating existing docs.
:param docs: a list of instances of SolrDoc.
:type server: list
:param commit: whether or not we should commit the documents.
:type server: boolean
"""
if not docs:
return
data = json.dumps(docs,
default=lambda obj: obj.isoformat()
if isinstance(obj, dt.datetime) else None)
params = {}
if commit:
params['commit'] = 'true'
return self.client.post(self._get_collection_url('update/json'),
params=params,
body=data)
def select(self,
query_dict,
groups=False,
facets=False,
stats=False,
**kwargs):
"""
Query documents from SOLR.
:param query_dict: a dict containing the query params to SOLR
:type query_dict: dict
:param metadata: whether or not solr metadata should be returned
:type metadata: boolean
:param kwargs: a dict of additional params for SOLR
:type kwargs: dict
:return: reformatted response from SOLR
:rtype: dict
"""
if kwargs:
query_dict.update(kwargs)
response = self.client.post(self._get_collection_url('select'),
body=json.dumps({'params': query_dict}))
data = {}
if groups and 'grouped' in response:
data['groups'] = response['grouped']
if facets and 'facet_counts' in response:
data['facets'] = response['facet_counts']
if stats and 'stats' in response:
data['stats'] = response['stats']
if 'response' in response and 'docs' in response['response']:
response_data = response['response']
data['docs'] = response_data['docs']
data['total'] = response_data.get('numFound', len(data['docs']))
return data
def delete(self, unique_key, unique_key_value, commit=False):
"""
Deleting a document from SOLR.
:param unique_key: the unique key for the doc to delete
:param unique_key_value: the value for the unique_key
:param commit: whether or not we should commit the documents.
:type server: boolean
"""
params = {}
if commit:
params['commit'] = 'true'
data = json.dumps(
{"delete": {
"query": "%s:%s" % (unique_key, unique_key_value)
}})
return self.client.post(self._get_collection_url('update/json'),
params=params,
body=data)
def commit(self):
"""
Hard commit documents to SOLR.
"""
params = {'commit': 'true'}
return self.client.post(self._get_collection_url('update/json'),
params=params)
def get_schema(self):
"""
Get the SOLR schema for the solr collection.
:return: the schema for the current collection
:rtype: dict
"""
response = self.client.get(self._get_collection_url('schema'))
return response.get('schema', {})
def add_schema_fields(self, fields):
"""
Add new fields to the schema of current collection
:param fields: a list of dicts of fields.
:type fields: list
"""
if not fields:
return
data = json.dumps(fields)
try:
return self.client.post(self._get_collection_url('schema/fields'),
body=data)
except solr_errors.SolrError as e:
raise solr_errors.SolrSchemaUpdateError(fields, message=e.args[0])
