def setUp(self):
super(TestElasticSearchProvider, self).setUp()
self.config = self.server.config
self.index = self.config.get('ELASTIC_SEARCH_INDEX')
self.ES = ElasticSearchProvider(self.config, self.db)
self.ES.activate_debug()
self.exception_generator = self.create_exception_generator()
def setUp(self):
super(TestElasticSearchProvider, self).setUp()
self.config = self.server.config
self.index = self.config.get('ELASTIC_SEARCH_INDEX')
self.ES = ElasticSearchProvider(self.config, self.db)
self.ES.activate_debug()
self.exception_generator = self.create_exception_generator()
class TestElasticSearchProvider(ApiTestCase):
def setUp(self):
super(TestElasticSearchProvider, self).setUp()
self.config = self.server.config
self.index = self.config.get('ELASTIC_SEARCH_INDEX')
self.ES = ElasticSearchProvider(self.config, self.db)
self.ES.activate_debug()
self.exception_generator = self.create_exception_generator()
def create_exception_generator(self):
yield ConnectionError('ConnectionError')
yield ElasticHttpNotFoundError('ElasticHttpNotFoundError')
yield InvalidJsonResponseError('InvalidJsonResponseError')
def mkfuture(self, result):
future = Future()
future.set_result(result)
return future
@patch('sqlalchemy.create_engine')
@patch('sqlalchemy.orm.scoped_session')
@patch('sqlalchemy.orm.sessionmaker')
def test_can_connect_to_db(self, sessionmaker_mock, scoped_session_mock, create_engine_mock):
self.ES.connect_to_db()
expect(create_engine_mock.call_count).to_equal(1)
expect(self.ES.db).to_be_instance_of(Mock)
create_engine_mock.assert_called_with(
self.config.get('SQLALCHEMY_CONNECTION_STRING'),
convert_unicode=True,
pool_size=1,
max_overflow=0,
echo=self.ES.debug
)
def test_can_assemble_inner_query(self):
expected = {
'match_all': {}
}
query = self.ES._assemble_inner_query()
expect(query).to_be_like(expected)
def test_can_assemble_inner_query_with_args(self):
expected = {
'prefix': {
'page_url': 'domain_url/page_filter'
}
}
query = self.ES._assemble_inner_query(
domain=Mock(url='domain_url'),
page_filter='page_filter'
)
expect(query).to_be_like(expected)
def test_can_assemble_outer_query(self):
filter_terms = ['term1', 'term2']
expected = {
'filtered': {
'query': 'inner_query',
'filter': {
'and': [{
'term': filter_term
} for filter_term in filter_terms]
}
}
}
query = self.ES._assemble_outer_query('inner_query', filter_terms)
expect(query).to_be_like(expected)
def test_can_assemble_filter_terms(self):
expected = [{'keys.id': 35}, {'domain_id': 359}]
query = self.ES._assemble_filter_terms(35, Mock(id=359))
expect(query).to_be_like(expected)
def test_can_gen_doc(self):
review = ReviewFactory.create(
is_active=True,
number_of_violations=6,
created_date=datetime(2014, 04, 15, 11, 44),
completed_date=datetime(2014, 04, 15, 11, 44),
)
expected = {
'keys': [{'id': violation.key_id} for violation in review.violations],
'uuid': str(review.uuid),
'completed_date': review.completed_date,
'violation_count': 6,
'page_id': review.page_id,
'page_uuid': str(review.page.uuid),
'page_url': review.page.url,
'page_last_review_date': review.page.last_review_date,
'domain_id': review.domain_id,
'domain_name': review.domain.name,
}
doc = self.ES.gen_doc(review)
expect(doc).to_be_like(expected)
@patch('time.sleep')
@patch('logging.error')
def test_can_index_review(self, logging_error_mock, time_sleep_mock):
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock())
for i in range(6):
review = Mock(page_id=0)
self.ES.index_review(review)
expect(self.ES.gen_doc.call_count).to_equal(6)
expect(self.ES.syncES.send_request.call_count).to_equal(6)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False
)
expect(logging_error_mock.called).to_be_false()
expect(time_sleep_mock.called).to_be_false()
@patch('time.sleep')
@patch('logging.error')
def test_try_index_review_up_to_three_times_and_raise_known_exceptions(self, logging_error_mock, time_sleep_mock):
def exception_raiser(*args, **kwargs):
raise next(self.exception_generator)
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock(side_effect=exception_raiser))
review = Mock(id=0, page_id=0)
try:
self.ES.index_review(review)
except:
err = sys.exc_info()[1]
expect(err).to_have_an_error_message_of('InvalidJsonResponseError')
else:
assert False, 'Should not have gotten this far'
expect(self.ES.gen_doc.call_count).to_equal(3)
expect(self.ES.syncES.send_request.call_count).to_equal(3)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False
)
expect(logging_error_mock.call_count).to_equal(3)
expect(time_sleep_mock.call_count).to_equal(3)
logging_error_mock.assert_has_calls([
call('Could not index review (review_id:0, page_id:0): ConnectionError'),
call('Could not index review (review_id:0, page_id:0): ElasticHttpNotFoundError'),
call('Could not index review (review_id:0, page_id:0): InvalidJsonResponseError')
])
@patch('time.sleep')
@patch('logging.error')
def test_try_index_review_just_once_and_raise_other_exceptions(self, logging_error_mock, time_sleep_mock):
def exception_raiser(*args, **kwargs):
raise AttributeError('AttributeError')
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock(side_effect=exception_raiser))
review = Mock(id=0, page_id=0)
try:
self.ES.index_review(review)
except:
err = sys.exc_info()[1]
expect(err).to_have_an_error_message_of('AttributeError')
else:
assert False, 'Should not have gotten this far'
expect(self.ES.gen_doc.call_count).to_equal(1)
expect(self.ES.syncES.send_request.call_count).to_equal(1)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False
)
expect(logging_error_mock.call_count).to_equal(0)
expect(time_sleep_mock.call_count).to_equal(0)
def test_can_index_reviews(self):
expected_body = '{"index":{"_type":"review","_id":0}}\n{"page_id":0}\n' * 5
self.ES.gen_doc = Mock(return_value={'page_id': 0})
self.ES.syncES = Mock(send_request=Mock())
pages = [Mock(last_review=0) for _ in range(100)]
self.ES.index_reviews(pages, len(pages), 5)
expect(self.ES.gen_doc.call_count).to_equal(100)
expect(self.ES.syncES.send_request.call_count).to_equal(20)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, '_bulk'],
body=expected_body,
encode_body=False
)
@gen_test
def test_can_get_by_violation_key_name(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(__getitem__=Mock())
hits = 6 * [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(len(hits))
expect(getitem_mock.__getitem__.call_count).to_equal(5 * len(hits))
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_server_erroneous_response(self):
def search_side_effect(*args, **kwargs):
error_mock = Mock(
error=Mock(message='error'),
body='error'
)
kwargs['callback'](error_mock)
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
callback_mock = Mock(side_effect=self.stop)
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: erroneous response (error [error])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_invalid_response(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise KeyError(args[0])
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(
__getitem__=Mock(side_effect=exception_raiser)
)
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(0)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: invalid response (<type \'exceptions.KeyError\'> [completed_date])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_invalid_response_again(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise TypeError('a float is required')
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(__getitem__=Mock())
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(
return_value='datetime',
side_effect=exception_raiser
)
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(1)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: invalid response (<type \'exceptions.TypeError\'> [a float is required])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(__getitem__=Mock(return_value=[]))
hits = 6 * [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(len(hits))
expect(getitem_mock.__getitem__.call_count).to_equal(5 * len(hits))
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_server_erroneous_response(self):
def search_side_effect(*args, **kwargs):
error_mock = Mock(
error=Mock(message='error'),
body='error'
)
kwargs['callback'](error_mock)
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
callback_mock = Mock(side_effect=self.stop)
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: erroneous response (error [error])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_invalid_response(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise KeyError(args[0])
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(
__getitem__=Mock(side_effect=exception_raiser)
)
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(0)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: invalid response (<type \'exceptions.KeyError\'> [completed_date])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_invalid_response_again(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise TypeError('a float is required')
self.ES.asyncES = Mock(
search=Mock(side_effect=search_side_effect)
)
getitem_mock = Mock(__getitem__=Mock(return_value=[]))
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(
return_value='datetime',
side_effect=exception_raiser
)
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads', return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime', utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock
)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(1)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with('ElasticSearchProvider: invalid response (<type \'exceptions.TypeError\'> [a float is required])')
callback_mock.assert_called_once_with(response)
class TestElasticSearchProvider(ApiTestCase):
def setUp(self):
super(TestElasticSearchProvider, self).setUp()
self.config = self.server.config
self.index = self.config.get('ELASTIC_SEARCH_INDEX')
self.ES = ElasticSearchProvider(self.config, self.db)
self.ES.activate_debug()
self.exception_generator = self.create_exception_generator()
def create_exception_generator(self):
yield ConnectionError('ConnectionError')
yield ElasticHttpNotFoundError('ElasticHttpNotFoundError')
yield InvalidJsonResponseError('InvalidJsonResponseError')
def mkfuture(self, result):
future = Future()
future.set_result(result)
return future
@patch('sqlalchemy.create_engine')
@patch('sqlalchemy.orm.scoped_session')
@patch('sqlalchemy.orm.sessionmaker')
def test_can_connect_to_db(self, sessionmaker_mock, scoped_session_mock,
create_engine_mock):
self.ES.connect_to_db()
expect(create_engine_mock.call_count).to_equal(1)
expect(self.ES.db).to_be_instance_of(Mock)
create_engine_mock.assert_called_with(
self.config.get('SQLALCHEMY_CONNECTION_STRING'),
convert_unicode=True,
pool_size=1,
max_overflow=0,
echo=self.ES.debug)
def test_can_assemble_inner_query(self):
expected = {'match_all': {}}
query = self.ES._assemble_inner_query()
expect(query).to_be_like(expected)
def test_can_assemble_inner_query_with_args(self):
expected = {'prefix': {'page_url': 'domain_url/page_filter'}}
query = self.ES._assemble_inner_query(domain=Mock(url='domain_url'),
page_filter='page_filter')
expect(query).to_be_like(expected)
def test_can_assemble_outer_query(self):
filter_terms = ['term1', 'term2']
expected = {
'filtered': {
'query': 'inner_query',
'filter': {
'and': [{
'term': filter_term
} for filter_term in filter_terms]
}
}
}
query = self.ES._assemble_outer_query('inner_query', filter_terms)
expect(query).to_be_like(expected)
def test_can_assemble_filter_terms(self):
expected = [{'keys.id': 35}, {'domain_id': 359}]
query = self.ES._assemble_filter_terms(35, Mock(id=359))
expect(query).to_be_like(expected)
def test_can_gen_doc(self):
review = ReviewFactory.create(
is_active=True,
number_of_violations=6,
created_date=datetime(2014, 04, 15, 11, 44),
completed_date=datetime(2014, 04, 15, 11, 44),
)
expected = {
'keys': [{
'id': violation.key_id
} for violation in review.violations],
'uuid': str(review.uuid),
'completed_date': review.completed_date,
'violation_count': 6,
'page_id': review.page_id,
'page_uuid': str(review.page.uuid),
'page_url': review.page.url,
'page_last_review_date': review.page.last_review_date,
'domain_id': review.domain_id,
'domain_name': review.domain.name,
}
doc = self.ES.gen_doc(review)
expect(doc).to_be_like(expected)
@patch('time.sleep')
@patch('logging.error')
def test_can_index_review(self, logging_error_mock, time_sleep_mock):
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock())
for i in range(6):
review = Mock(page_id=0)
self.ES.index_review(review)
expect(self.ES.gen_doc.call_count).to_equal(6)
expect(self.ES.syncES.send_request.call_count).to_equal(6)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False)
expect(logging_error_mock.called).to_be_false()
expect(time_sleep_mock.called).to_be_false()
@patch('time.sleep')
@patch('logging.error')
def test_try_index_review_up_to_three_times_and_raise_known_exceptions(
self, logging_error_mock, time_sleep_mock):
def exception_raiser(*args, **kwargs):
raise next(self.exception_generator)
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock(side_effect=exception_raiser))
review = Mock(id=0, page_id=0)
try:
self.ES.index_review(review)
except:
err = sys.exc_info()[1]
expect(err).to_have_an_error_message_of('InvalidJsonResponseError')
else:
assert False, 'Should not have gotten this far'
expect(self.ES.gen_doc.call_count).to_equal(3)
expect(self.ES.syncES.send_request.call_count).to_equal(3)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False)
expect(logging_error_mock.call_count).to_equal(3)
expect(time_sleep_mock.call_count).to_equal(3)
logging_error_mock.assert_has_calls([
call(
'Could not index review (review_id:0, page_id:0): ConnectionError'
),
call(
'Could not index review (review_id:0, page_id:0): ElasticHttpNotFoundError'
),
call(
'Could not index review (review_id:0, page_id:0): InvalidJsonResponseError'
)
])
@patch('time.sleep')
@patch('logging.error')
def test_try_index_review_just_once_and_raise_other_exceptions(
self, logging_error_mock, time_sleep_mock):
def exception_raiser(*args, **kwargs):
raise AttributeError('AttributeError')
self.ES.gen_doc = Mock(return_value=0)
self.ES.syncES = Mock(send_request=Mock(side_effect=exception_raiser))
review = Mock(id=0, page_id=0)
try:
self.ES.index_review(review)
except:
err = sys.exc_info()[1]
expect(err).to_have_an_error_message_of('AttributeError')
else:
assert False, 'Should not have gotten this far'
expect(self.ES.gen_doc.call_count).to_equal(1)
expect(self.ES.syncES.send_request.call_count).to_equal(1)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, 'review', 0],
body='0',
encode_body=False)
expect(logging_error_mock.call_count).to_equal(0)
expect(time_sleep_mock.call_count).to_equal(0)
def test_can_index_reviews(self):
expected_body = '{"index":{"_type":"review","_id":0}}\n{"page_id":0}\n' * 5
self.ES.gen_doc = Mock(return_value={'page_id': 0})
self.ES.syncES = Mock(send_request=Mock())
pages = [Mock(last_review=0) for _ in range(100)]
self.ES.index_reviews(pages, len(pages), 5)
expect(self.ES.gen_doc.call_count).to_equal(100)
expect(self.ES.syncES.send_request.call_count).to_equal(20)
self.ES.syncES.send_request.assert_called_with(
method='POST',
path_components=[self.index, '_bulk'],
body=expected_body,
encode_body=False)
@gen_test
def test_can_get_by_violation_key_name(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock())
hits = 6 * [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(len(hits))
expect(getitem_mock.__getitem__.call_count).to_equal(5 * len(hits))
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_server_erroneous_response(
self):
def search_side_effect(*args, **kwargs):
error_mock = Mock(error=Mock(message='error'), body='error')
kwargs['callback'](error_mock)
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
callback_mock = Mock(side_effect=self.stop)
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1, current_page=1, page_size=10, callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: erroneous response (error [error])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_invalid_response(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise KeyError(args[0])
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock(side_effect=exception_raiser))
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(0)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: invalid response (<type \'exceptions.KeyError\'> [completed_date])'
)
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_by_violation_key_name_with_invalid_response_again(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise TypeError('a float is required')
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock())
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime',
side_effect=exception_raiser)
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_by_violation_key_name(
key_id=1,
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(1)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: invalid response (<type \'exceptions.TypeError\'> [a float is required])'
)
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock(return_value=[]))
hits = 6 * [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(len(hits))
expect(getitem_mock.__getitem__.call_count).to_equal(5 * len(hits))
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_server_erroneous_response(
self):
def search_side_effect(*args, **kwargs):
error_mock = Mock(error=Mock(message='error'), body='error')
kwargs['callback'](error_mock)
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
callback_mock = Mock(side_effect=self.stop)
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: erroneous response (error [error])')
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_invalid_response(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise KeyError(args[0])
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock(side_effect=exception_raiser))
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime')
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(0)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: invalid response (<type \'exceptions.KeyError\'> [completed_date])'
)
callback_mock.assert_called_once_with(response)
@gen_test
def test_can_get_domain_active_reviews_with_invalid_response_again(self):
def search_side_effect(*args, **kwargs):
kwargs['callback'](Mock(error=None))
def exception_raiser(*args, **kwargs):
raise TypeError('a float is required')
self.ES.asyncES = Mock(search=Mock(side_effect=search_side_effect))
getitem_mock = Mock(__getitem__=Mock(return_value=[]))
hits = [{'_source': getitem_mock}]
fake_body = {'total': len(hits), 'hits': {'hits': hits}}
utcfromtimestamp_mock = Mock(return_value='datetime',
side_effect=exception_raiser)
callback_mock = Mock(side_effect=self.stop)
with patch('holmes.search_providers.elastic.loads',
return_value=fake_body):
with patch('holmes.search_providers.elastic.datetime',
utcfromtimestamp=utcfromtimestamp_mock):
with patch('logging.error') as logging_error_mock:
response = yield self.ES.get_domain_active_reviews(
domain=Mock(id=359),
current_page=1,
page_size=10,
callback=callback_mock)
expect(self.ES.asyncES.search.call_count).to_equal(1)
expect(utcfromtimestamp_mock.call_count).to_equal(1)
expect(getitem_mock.__getitem__.call_count).to_equal(1)
logging_error_mock.assert_called_once_with(
'ElasticSearchProvider: invalid response (<type \'exceptions.TypeError\'> [a float is required])'
)
callback_mock.assert_called_once_with(response)
def use_elastic_search_provider(self):
from holmes.search_providers.elastic import ElasticSearchProvider
self.server.application.search_provider = ElasticSearchProvider(
config=self.server.application.config,
db=self.db,
io_loop=self.io_loop)
