def put(self, unauthorized=False):
if self.request.body == "unauthorized":
raise ClientHTTPError(401, "http error: unauthorized, back off")
elif self.request.body == "400":
_buffer = StringIO.StringIO()
_buffer.write("Malformed query")
response = HTTPResponse(self.request, 400, buffer=_buffer, effective_url="/a")
raise ClientHTTPError(400, message="Bad request", response=response)
elif self.request.body == "500":
raise HTTPError(500, "Internal Virtuoso Error")
else:
raise CurlError(500, "Virtuoso Down on port 8890")
class SearchEngineTestCase(TestCase):
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_request_url_all_none(self):
expected_url = "http://esearch.host/semantica.*/_search"
response = search_engine._build_elasticsearch_request_url(None)
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_request_url(self):
expected_url = "http://esearch.host/semantica.glb/_search"
response = search_engine._build_elasticsearch_request_url(["semantica.glb"])
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_default_usage(self):
expected_url = "http://esearch.host/_analyze?text=something"
response = search_engine._build_elasticsearch_analyze_url(target="something")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_special_characters(self):
expected_url = "http://esearch.host/_analyze?text=%C5%9A%E1%B9%95%C3%A9c%C3%AC%C3%A3l+ch%C3%A2rs"
response = search_engine._build_elasticsearch_analyze_url(target="Śṕécìãl chârs")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_special_characters_encoded(self):
expected_url = "http://esearch.host/_analyze?text=galv%C3%A3o"
response = search_engine._build_elasticsearch_analyze_url(target=u"galv\xe3o")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_with_multiple_indexes(self):
expected_url = "http://esearch.host/_analyze?text=anything"
response = search_engine._build_elasticsearch_analyze_url(target="anything")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_with_non_default_analyzer(self):
expected_url = "http://esearch.host/_analyze?text=dummything"
response = search_engine._build_elasticsearch_analyze_url(target="dummything")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine.ELASTICSEARCH_ENDPOINT", "esearch.host")
def test_build_elasticsearch_analyze_url_with_text_that_needs_scaping(self):
expected_url = "http://esearch.host/_analyze?text=text+with+spaces"
response = search_engine._build_elasticsearch_analyze_url(target="text with spaces")
self.assertEquals(expected_url, response)
@patch("brainiak.search_engine._get_response",
return_value=MockResponse('{}', 200))
def test_run_analyze(self, request_mock):
response = search_engine.run_analyze(target="sometext")
self.assertEquals(response, {})
call_args = request_mock.call_args[0][0]
self.assertEqual(call_args["url"], u'http://*****:*****@patch("brainiak.search_engine._get_response",
return_value=MockResponse("{}", 200))
def test_run_search(self, request_mock):
response = search_engine.run_search(
body={},
indexes=["index1"]
)
self.assertEquals(response, {})
call_args = request_mock.call_args[0][0]
self.assertEqual(call_args["url"], u'http://*****:*****@patch("brainiak.search_engine._do_request",
return_value=MockResponse("{}", 200))
def test_save_instance_return_200(self, request_mock):
response = search_engine.save_instance({"test": "a"}, "index", "type", "id")
expected_url = u"http://*****:*****@patch("brainiak.search_engine._get_response",
return_value=MockResponse('{"error"}', 400))
def test_save_instance_return_400(self, request_mock):
expected_msg = 'Error on Elastic Search when saving an instance.\n PUT http://localhost:9200/index/type/id - 400\n Body: {"error"}'
try:
search_engine.save_instance({"error": "json not in conformance with mapping, ES returns 400"}, "index", "type", "id")
except ClientHTTPError as e:
self.assertEqual(e.status_code, 400)
self.assertEqual(e.log_message, expected_msg)
@patch("brainiak.search_engine._get_response",
return_value=MockResponse('{"my_instance": 123}', 200))
def test_get_instance_return_200(self, request_mock):
instance = search_engine.get_instance("index", "type", "id")
self.assertEqual(instance, {"my_instance": 123})
@patch("brainiak.search_engine._get_response",
return_value=None)
def test_get_instance_returns_none(self, request_mock):
self.assertIsNone(search_engine.get_instance("index", "type", "id"))
@patch("brainiak.search_engine._get_response",
side_effect=ClientHTTPError(500, message="xubi"))
def test_get_instance_return_500(self, request_mock):
expected_msg = 'HTTP 500: xubi'
try:
search_engine.get_instance("index", "type", "id")
except ClientHTTPError as e:
self.assertEqual(e.code, 500)
self.assertEqual(e.message, expected_msg)
@patch("brainiak.search_engine._get_response",
return_value=MockResponse('{"my_instance": 123}', 200))
def test_delete_instance_true(self, request_mock):
self.assertTrue(search_engine.delete_instance("index", "type", "id"))
@patch("brainiak.search_engine._get_response",
return_value=None)
def test_delete_instance_false(self, request_mock):
self.assertFalse(search_engine.delete_instance("index", "type", "id"))
@patch("brainiak.search_engine._do_request",
return_value=MockResponse("{}", 200))
def test_get_response(self, mock_do_request):
expected_code = 200
response = search_engine._get_response({})
self.assertEqual(response.code, expected_code)
@patch("brainiak.search_engine._do_request",
side_effect=ClientHTTPError(400, message="error"))
def test_get_response_raises_ClientHTTPError(self, mock_do_request):
expected_code = 400
expected_msg = "HTTP 400: error"
try:
search_engine._get_response({})
except ClientHTTPError as e:
self.assertEqual(e.code, expected_code)
self.assertEqual(e.message, expected_msg)
@patch("brainiak.search_engine._do_request",
side_effect=ClientHTTPError(404, message="error"))
def test_get_response_404_returns_none(self, mock_do_request):
self.assertIsNone(search_engine._get_response({}))
@patch("brainiak.log.logger.info")
@patch("brainiak.search_engine.greenlet_fetch",
return_value=MockResponse("{}", 200))
def test_do_request(self, fetch_mock, mock_log_info):
expected_msg = u'ELASTICSEARCH - GET - http://a-url.com - 200 - [time: 0] - REQUEST BODY - - RESPONSE BODY - {}'
expected_code = 200
# mock time.time
time_patcher = patch.object(
search_engine, 'time',
Mock(wraps=time)
)
mocked_time = time_patcher.start()
mocked_time.time.return_value = 0
self.addCleanup(time_patcher.stop)
# end mock time.time
request_params = {
"url": "http://a-url.com",
"method": "GET",
}
response = search_engine._do_request(request_params)
self.assertEquals(response.code, expected_code)
mock_log_info.assert_called_with(expected_msg)
class TriplestoreTestCase(unittest.TestCase):
maxDiff = None
TRIPLESTORE_CONFIG = {
"app_name": "Brainiak",
"url": "url",
"auth_mode": "digest",
"auth_username": "******",
"auth_password": "******"
}
EXAMPLE_QUERY = "SELECT * {?s a ?o}"
EXAMPLE_QUERY_URL_ENCODED = u"query=SELECT+%2A+%7B%3Fs+a+%3Fo%7D&format=application%2Fsparql-results%2Bjson"
@patch("brainiak.triplestore.log.logger")
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch("brainiak.triplestore.requests.request",
return_value=MockResponse())
def test_both_without_auth_and_with_auth_work(self, mock_request,
mock_parse_section, log):
received_msg = triplestore.status()
msg1 = 'Virtuoso connection authenticated [USER:PASSWORD] | SUCCEED | url'
msg2 = 'Virtuoso connection not-authenticated | SUCCEED | url'
expected_msg = "<br>".join([msg1, msg2])
self.assertEqual(received_msg, expected_msg)
@patch("brainiak.triplestore.log.logger")
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch("brainiak.triplestore.requests.request",
side_effect=[MockResponse(401), MockResponse()])
def test_without_auth_works_but_with_auth_doesnt(self, mock_parse_section,
mock_request, mock_log):
received_msg = triplestore.status()
msg1 = "Virtuoso connection authenticated [USER:PASSWORD] | FAILED | url | Status code: 401. Message: "
msg2 = "Virtuoso connection not-authenticated | SUCCEED | url"
expected_msg = "<br>".join([msg1, msg2])
self.assertEqual(received_msg, expected_msg)
@patch("brainiak.triplestore.log.logger")
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch("brainiak.triplestore.requests.request",
side_effect=[MockResponse(), MockResponse(401)])
def test_without_auth_doesnt_work_but_with_auth_works(
self, mock_request, mock_parse_section, mock_log):
received_msg = triplestore.status()
msg1 = "Virtuoso connection authenticated [USER:PASSWORD] | SUCCEED | url"
msg2 = "Virtuoso connection not-authenticated | FAILED | url | Status code: 401. Message: "
expected_msg = "<br>".join([msg1, msg2])
self.assertEqual(received_msg, expected_msg)
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch("brainiak.triplestore.requests.request",
return_value=MockResponse(401))
def _test_both_without_auth_and_with_auth_dont_work(
self, mock_request, mock_parse_section):
received_msg = triplestore.status()
msg1 = "Virtuoso connection authenticated [USER:PASSWORD] | FAILED | url | Status code: 401. Message: "
msg2 = "Virtuoso connection not-authenticated | FAILED | url | Status code: 401. Message: "
expected_msg = "<br>".join([msg1, msg2])
self.assertEqual(received_msg, expected_msg)
def test_process_triplestore_response_async_true(self):
expected = {"a": "json string"}
class TornadoHTTPResponse:
body = '{"a": "json string"}'
tornado_response = TornadoHTTPResponse()
response = triplestore._process_json_triplestore_response(
tornado_response)
self.assertEqual(expected, response)
def test_process_triplestore_response_async_false(self):
expected = {"a": "json string"}
class RequestsHTTPResponse:
def json(self):
return {"a": "json string"}
tornado_response = RequestsHTTPResponse()
response = triplestore._process_json_triplestore_response(
tornado_response, async=False)
self.assertEqual(response, expected)
def test_process_triplestore_response_async_false_invalid_json(self):
class RequestsHTTPResponse:
def json(self):
raise simplejson.JSONDecodeError("", "", 10)
text = ""
tornado_response = RequestsHTTPResponse()
self.assertRaises(ClientHTTPError,
triplestore._process_json_triplestore_response,
tornado_response,
async=False)
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch('brainiak.triplestore.greenlet_fetch',
side_effect=ClientHTTPError(401, message=""))
def test_query_sparql_with_http_error_401(self, run_query,
mock_parse_section):
request_params = {"url": "http://aa"}
self.assertRaises(HTTPError,
triplestore.do_run_query,
request_params,
async=True)
@patch("brainiak.triplestore.parse_section",
return_value={
"auth_username": "******",
"auth_password": "******",
"url": "url"
})
@patch('brainiak.triplestore.greenlet_fetch',
side_effect=ClientHTTPError(500, message=""))
def test_query_sparql_with_http_error_500(self, run_query,
mock_parse_section):
request_params = {"url": "http://aa"}
self.assertRaises(ClientHTTPError,
triplestore.do_run_query,
request_params,
async=True)
def test_build_request_params_for_async_query(self):
expected_request_for_tornado = {
"headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"method": triplestore.DEFAULT_HTTP_METHOD,
"body": self.EXAMPLE_QUERY_URL_ENCODED
}
expected_request_for_tornado.update(self.TRIPLESTORE_CONFIG)
response = triplestore._build_request_params(self.EXAMPLE_QUERY,
self.TRIPLESTORE_CONFIG,
async=True)
self.assertEqual(response, expected_request_for_tornado)
def test_build_request_params_for_sync_query(self):
expected_request_for_requests = {
"headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"method": triplestore.DEFAULT_HTTP_METHOD,
"data": {
"query": self.EXAMPLE_QUERY,
'format': triplestore.DEFAULT_RESPONSE_FORMAT
}
}
expected_request_for_requests.update(self.TRIPLESTORE_CONFIG)
response = triplestore._build_request_params(self.EXAMPLE_QUERY,
self.TRIPLESTORE_CONFIG,
async=False)
response.pop("auth") # object created inside _build_request_params
self.assertEqual(response, expected_request_for_requests)
@patch('brainiak.triplestore.log.logger')
@patch('brainiak.triplestore.do_run_query',
return_value=(MockResponse(), 0))
def test_query_sparql_without_error(self, run_query, mock_log):
response = triplestore.query_sparql("", self.TRIPLESTORE_CONFIG)
self.assertEqual(response, {})
@patch('brainiak.triplestore.greenlet_fetch', return_value=MockResponse())
@patch('brainiak.triplestore.log')
def test_query_sparql_with_valid_credential(self, mocked_log,
greenlet_fetch):
response = triplestore.query_sparql("", triplestore_config)
self.assertEqual(greenlet_fetch.call_count, 1)
self.assertEqual(response, {})
log_msg = format_post % log_params
log.logger.info(log_msg)
def _process_json_triplestore_response(response, async=True):
"""
Returns a python dict with triplestore response.
Unifying tornado and requests response.
"""
if async:
result_dict = json.loads(str(response.body.decode("utf-8")))
else:
try:
result_dict = response.json()
except JSONDecodeError:
raise ClientHTTPError(
400, JSON_DECODE_ERROR_MESSAGE.format(response.text))
return result_dict
def query_sparql(query, triplestore_config, async=True):
"""
Simple interface that given a SPARQL query string returns a string representing a SPARQL results bindings
in JSON format. For now it only works with Virtuoso, but in futurw we intend to support other databases
that are SPARQL 1.1 complaint (including SPARQL result bindings format).
"""
request_params = _build_request_params(query, triplestore_config, async)
log_params = copy.copy(request_params)
response, time_diff = do_run_query(request_params, async)
log_params["query"] = str(query)