def test_use_of_custom_http_clients(self):
mocked_http_client = Mock()
mocked_http_client.fetch = Mock()
es_connection = ESConnection("localhost",
"9200",
self.io_loop,
custom_client=mocked_http_client)
es_connection.search(callback=self.stop,
source={"query": {"term": {"ID": "171171"}}},
type="materia", index="teste")
mocked_http_client.fetch.assert_called()
def test_use_of_custom_http_clients(self):
mocked_http_client = Mock()
mocked_http_client.fetch = Mock()
es_connection = ESConnection("localhost",
"9200",
self.io_loop,
custom_client=mocked_http_client)
es_connection.search(callback=self.stop,
source={"query": {"term": {"ID": "171171"}}},
type="materia", index="teste")
mocked_http_client.fetch.assert_called()
class TestESConnectionWithTornadoGen(AsyncTestCase):
def setUp(self):
super(TestESConnectionWithTornadoGen, self).setUp()
self.es_connection = ESConnection("localhost", "9200", self.io_loop)
def tearDown(self):
if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
self.io_loop.close(all_fds=True)
super(AsyncTestCase, self).tearDown()
@gen_test
def test_simple_search(self):
response = yield self.es_connection.get_by_path("/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'http://localhost/noticia/2/fast')
@gen_test
def test_search_for_specific_type_with_query(self):
response = yield self.es_connection.search(
source={"query": {"term": {"ID": "171171"}}},
type="materia", index="teste"
)
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')
@gen_test
def test_search_specific_index(self):
response = yield self.es_connection.search(index="outroteste")
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 14)
@gen_test
def test_search_apecific_type(self):
response = yield self.es_connection.search(type='galeria')
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 2)
@gen_test
def test_should_access_specific_document_using_tornado_gen(self):
response = yield self.es_connection.get(index="teste", type="materia", uid="171171")
response = response["_source"]
self.assertEqual(response['Portal'], "G1")
self.assertEqual(response['Macrotema'], "Noticias")
@gen_test
def test_should_make_two_searches(self):
self._make_multisearch()
response = yield self.es_connection.apply_search()
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'], "171171")
self.assertFalse("hits" in response['responses'][1])
@gen_test
def test_should_clean_search_list_after_search(self):
self._make_multisearch()
response = yield self.es_connection.apply_search()
response = self._verify_status_code_and_return_response(response)
self.assertListEqual([], self.es_connection.bulk.bulk_list)
@gen_test
def test_can_put_and_delete_document(self):
try:
doc_id = str(uuid4())
response = yield self.es_connection.put("test", "document", doc_id, {
"test": "document",
"other": "property"
}, parameters={'refresh': True})
response_dict = self._verify_status_code_and_return_response(response)
self.assertEqual(response_dict['_index'], 'test')
self.assertEqual(response_dict['_type'], 'document')
self.assertEqual(response_dict['_id'], doc_id)
self.assertIn('refresh=True', response.request.url)
finally:
response = yield self.es_connection.delete("test", "document", doc_id,
parameters={'refresh': True})
response = self._verify_status_code_and_return_response(response)
self.assertTrue(response['found'])
self.assertEqual(response['_index'], 'test')
self.assertEqual(response['_type'], 'document')
self.assertEqual(response['_id'], doc_id)
@gen_test
def test_count_specific_index(self):
response = yield self.es_connection.count(index="outroteste")
self.assertCount(response, 14)
@gen_test
def test_count_specific_type(self):
response = yield self.es_connection.count(type='galeria')
self.assertCount(response, 2)
@gen_test
def test_count_specific_query(self):
source = {"query": {"term": {"_id": "171171"}}}
response = yield self.es_connection.count(source=source)
self.assertCount(response, 1)
@gen_test
def test_count_specific_query_with_parameters(self):
source = {"query": {"term": {"_id": "171171"}}}
parameters = {'refresh': True}
response = yield self.es_connection.count(callback=self.stop, source=source, parameters=parameters)
self.assertCount(response, 1)
self.assertTrue(response.request.url.endswith('_count?refresh=True'))
@gen_test
def test_count_specific_query_with_many_parameters(self):
source = {"query": {"term": {"_id": "171171"}}}
parameters = {'df': '_id', 'test': True}
response = yield self.es_connection.count(callback=self.stop, source=source, parameters=parameters)
self.assertCount(response, 1)
self.assertTrue('df=_id' in response.request.url)
self.assertTrue('test=True' in response.request.url)
def assertCount(self, response, count):
response_dict = self._verify_status_code_and_return_response(response)
self.assertEqual(response_dict["count"], count)
def _make_multisearch(self):
source = {"query": {"term": {"_id": "171171"}}}
self.es_connection.multi_search(index="teste", source=source)
source = {"query": {"term": {"_id": "101010"}}}
self.es_connection.multi_search(index="neverEndIndex", source=source)
def _verify_status_code_and_return_response(self, response):
self.assertTrue(response.code in [200, 201], "Wrong response code: %d." % response.code)
response = escape.json_decode(response.body)
return response
class ElasticSearchProvider(SearchProvider):
def __init__(self, config, db=None, authnz_wrapper=None, io_loop=None):
self.debug = False
self.config = config
if db is not None:
self.db = db
self.syncES = ElasticSearch(
'%(ELASTIC_SEARCH_PROTOCOL)s://%(ELASTIC_SEARCH_HOST)s:%(ELASTIC_SEARCH_PORT)s' % config
)
self.asyncES = ESConnection(
host=config.get('ELASTIC_SEARCH_HOST'),
port=config.get('ELASTIC_SEARCH_PORT'),
io_loop=io_loop,
protocol=config.get('ELASTIC_SEARCH_PROTOCOL'),
)
self.index = config.get('ELASTIC_SEARCH_INDEX')
self.max_retries = config.get('ELASTIC_SEARCH_MAX_RETRIES')
def activate_debug(self):
self.debug = True
def connect_to_db(self):
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
conn_string = self.config.get('SQLALCHEMY_CONNECTION_STRING')
engine = create_engine(
conn_string,
convert_unicode=True,
pool_size=1,
max_overflow=0,
echo=self.debug
)
maker = sessionmaker(bind=engine, autoflush=True)
self.db = scoped_session(maker)
def _assemble_inner_query(self, domain=None, page_filter=None):
if page_filter and domain:
page_prefix = '%s/%s' % (domain.url, page_filter)
else:
page_prefix = None
if page_prefix:
return {
'prefix': {
'page_url': page_prefix
}
}
else:
return {
'match_all': {}
}
def _assemble_outer_query(self, inner_query, filter_terms):
return {
'filtered': {
'query': inner_query,
'filter': {
'and': [{
'term': filter_term
} for filter_term in filter_terms]
}
}
}
def _assemble_filter_terms(self, key_id=None, domain=None):
filter_terms = []
if key_id:
filter_terms.append({'keys.id': key_id})
if domain:
filter_terms.append({'domain_id': domain.id})
return filter_terms
def gen_doc(self, review):
return {
'keys': [{'id': violation.key_id} for violation in review.violations],
'uuid': str(review.uuid),
'completed_date': review.completed_date,
'violation_count': review.violation_count,
'page_id': review.page_id,
'page_uuid': str(review.page.uuid),
'page_url': review.page.url,
'page_last_review_date': review.page.last_review_date,
'domain_id': review.domain_id,
'domain_name': review.domain.name,
}
def index_review(self, review):
for attempt in range(self.max_retries):
try:
self.syncES.send_request(
method='POST',
path_components=[self.index, 'review', review.page_id],
body=dumps(self.gen_doc(review)),
encode_body=False
)
break
except (Timeout, ConnectionError, ElasticHttpError, InvalidJsonResponseError) as e:
values = review.id, review.page_id, str(e)
logging.error('Could not index review (review_id:{0}, page_id:{1}): {2}'.format(*values))
time.sleep(1)
if attempt >= self.max_retries - 1:
raise
else:
raise
def index_reviews(self, reviewed_pages, reviews_count, batch_size):
action = {'index': {'_type': 'review'}}
for i in range(0, reviews_count, batch_size):
body_bits = []
for page in reviewed_pages[i:i + batch_size]:
doc = self.gen_doc(page.last_review)
action['index']['_id'] = doc['page_id']
body_bits.append(dumps(action))
body_bits.append(dumps(doc))
# Yes, that trailing newline IS necessary
body = '\n'.join(body_bits) + '\n'
self.syncES.send_request(
method='POST',
path_components=[self.index, '_bulk'],
body=body,
encode_body=False
)
logging.info('Done!')
@return_future
def get_by_violation_key_name(self, key_id, current_page=1, page_size=10, domain=None, page_filter=None, callback=None):
def treat_response(response):
if response.error is None:
try:
hits = loads(response.body).get('hits', {'hits': []})
reviews_data = []
for hit in hits['hits']:
completedAt = datetime.utcfromtimestamp(hit['_source']['completed_date'])
reviews_data.append({
'uuid': hit['_source']['uuid'],
'page': {
'uuid': hit['_source']['page_uuid'],
'url': hit['_source']['page_url'],
'completedAt': completedAt
},
'domain': hit['_source']['domain_name']
})
reviews_count = hits.get('total', 0)
callback({
'reviews': reviews_data,
'reviewsCount': reviews_count
})
except Exception as e:
reason = 'ElasticSearchProvider: invalid response (%s [%s])' % (type(e), e.message)
logging.error(reason)
callback({'error': {'status_code': 500, 'reason': reason}})
else:
reason = 'ElasticSearchProvider: erroneous response (%s [%s])' % (response.error.message, response.body)
logging.error(reason)
callback({'error': {'status_code': 500, 'reason': reason}})
inner_query = self._assemble_inner_query(domain, page_filter)
filter_terms = self._assemble_filter_terms(key_id, domain)
query = self._assemble_outer_query(inner_query, filter_terms)
sort_ = [{
'completed_date': {
'order': 'desc'
}
}, {
'violation_count': {
'order': 'desc'
}
}]
source = {'query': query, 'sort': sort_}
self.asyncES.search(
callback=treat_response,
index=self.index,
type='review',
source=source,
page=current_page,
size=page_size,
)
@return_future
def get_domain_active_reviews(self, domain, current_page=1, page_size=10, page_filter=None, callback=None):
def treat_response(response):
if response.error is None:
try:
hits = loads(response.body).get('hits', {'hits': []})
pages = []
for hit in hits['hits']:
completedAt = datetime.utcfromtimestamp(hit['_source']['completed_date'])
pages.append({
'url': hit['_source']['page_url'],
'uuid': hit['_source']['page_uuid'],
'violationCount': len(hit['_source']['keys']),
'completedAt': completedAt,
'reviewId': hit['_source']['uuid']
})
reviews_count = hits.get('total', 0)
callback({
'reviewsCount': reviews_count,
'pages': pages
})
except Exception as e:
reason = 'ElasticSearchProvider: invalid response (%s [%s])' % (type(e), e.message)
logging.error(reason)
callback({'error': {'status_code': 500, 'reason': reason}})
else:
reason = 'ElasticSearchProvider: erroneous response (%s [%s])' % (response.error.message, response.body)
logging.error(reason)
callback({'error': {'status_code': 500, 'reason': reason}})
inner_query = self._assemble_inner_query(domain=domain, page_filter=page_filter)
filter_terms = self._assemble_filter_terms(domain=domain)
query = self._assemble_outer_query(inner_query, filter_terms)
sort_ = [{
'violation_count': {
'order': 'desc'
}
}, {
'completed_date': {
'order': 'desc'
}
}]
source = {'query': query, 'sort': sort_}
self.asyncES.search(
callback=treat_response,
index=self.index,
type='review',
source=source,
page=current_page,
size=page_size,
)
def refresh(self):
try:
self.syncES.refresh(index=self.index)
except Exception as e:
logging.error('Could not refresh index (%s)' % e)
def get_index_settings(cls):
return {
'index': {
'number_of_shards': 4
}
}
def get_index_mapping(cls):
return {
'review': {
'properties': {
'keys': {
'properties': {
'id': {
'type': 'integer'
}
}
},
'uuid': {
'type': 'string',
'index': 'not_analyzed'
},
'completed_date': {
'type': 'integer'
},
'violation_count': {
'type': 'float'
},
'page_id': {
'type': 'integer'
},
'page_uuid': {
'type': 'string',
'index': 'not_analyzed'
},
'page_url': {
'type': 'string',
'index': 'not_analyzed'
},
'page_last_review_date': {
'type': 'integer'
},
'domain_id': {
'type': 'integer'
},
'domain_name': {
'type': 'string',
'index': 'not_analyzed'
}
}
}
}
def setup_index(self):
try:
settings = self.get_index_settings()
self.syncES.create_index(index=self.index, settings=settings)
mapping = self.get_index_mapping()
self.syncES.put_mapping(index=self.index, doc_type='review', mapping=mapping)
logging.info('Index %s created.' % self.index)
except Exception as e:
raise e
def delete_index(self):
try:
self.syncES.delete_index(index=self.index)
logging.info('Index %s deleted.' % self.index)
except Exception as e:
raise e
def _get_max_page_id_from_index(self, must_have_domain_name=False):
if must_have_domain_name:
inner_query = {
'constant_score': {
'filter': {
'not': {
'missing': {
'field': 'domain_name'
}
}
}
}
}
else:
inner_query = {
'match_all': {}
}
query = {
'query': inner_query,
'sort': [{
'page_id': {
'order': 'desc'
}
}]
}
results = self.syncES.search(query, index=self.index, doc_type='review')
if results['hits']['total'] > 0:
return results['hits']['hits'][0]['_id'] or 0
return 0
def index_all_reviews(self, keys=None, batch_size=200, replace=False):
logging.info('Querying database...')
self.connect_to_db()
if keys is not None:
keys = [k.id for k in self.db.query(Key.id).filter(Key.name.in_(keys)).all()]
try:
max_page_id = self._get_max_page_id_from_index(must_have_domain_name=True)
except Exception:
logging.error('Could not retrieve max page_id! Use with --replace (with caution)')
return
def apply_filters(query):
if keys is not None:
query = query \
.filter(Violation.review_id == Page.last_review_id) \
.filter(Violation.key_id.in_(keys))
if not replace:
query = query.filter(Page.id > max_page_id)
return query.filter(Page.last_review_id != None)
reviews_count = apply_filters(self.db.query(func.count(Page))).scalar()
query = self.db.query(Page).options(joinedload('last_review'))
reviewed_pages = apply_filters(query).order_by(Page.id.asc())
logging.info('Indexing %d reviews...' % reviews_count)
self.index_reviews(reviewed_pages, reviews_count, batch_size)
@classmethod
def new_instance(cls, config):
return ElasticSearchProvider(config)
@classmethod
def main(cls):
import sys
parser = cls.argparser()
args = parser.parse_args()
config = {}
host = None
port = None
index = None
es = None
levels = ['ERROR', 'WARNING', 'INFO', 'DEBUG']
log_level = levels[args.verbose]
logging.basicConfig(level=getattr(logging, log_level), format='%(levelname)s - %(message)s')
if not (args.create or args.recreate or args.delete or args.keys or args.all_keys):
parser.print_help()
sys.exit(1)
if args.conf:
from derpconf.config import ConfigurationError
from holmes.config import Config
try:
config = Config().load(args.conf[0])
host = config['ELASTIC_SEARCH_HOST']
port = config['ELASTIC_SEARCH_PORT']
index = config['ELASTIC_SEARCH_INDEX']
except ConfigurationError:
logging.error('Could not load config! Use --conf conf_file')
sys.exit(1)
except KeyError:
logging.error('Could not parse config! Check it\'s contents')
sys.exit(1)
if args.server:
try:
host, port = args.server[0].split(':')
config['ELASTIC_SEARCH_HOST'] = host
config['ELASTIC_SEARCH_PORT'] = port
except Exception:
logging.error('Could not parse server host and port! Use --server host:port')
sys.exit(1)
if args.index:
index = args.index[0]
config['ELASTIC_SEARCH_INDEX'] = index
from pyelasticsearch.exceptions import IndexAlreadyExistsError, ElasticHttpNotFoundError, InvalidJsonResponseError
from requests.exceptions import ConnectionError
try:
if args.create or args.recreate or args.delete:
if host is None or port is None:
logging.error('Need either a host and port or a config file to perform such operation!')
sys.exit(1)
if index is None:
logging.error('Need either an index name or a config file to perform such operation!')
sys.exit(1)
else:
es = cls.new_instance(config)
if args.recreate or args.delete:
try:
es.delete_index()
except ElasticHttpNotFoundError:
pass
except InvalidJsonResponseError as e:
logging.error('Invalid response! Reason: %s' % e)
sys.exit(1)
if args.create or args.recreate:
es.setup_index()
if args.keys or args.all_keys:
if config is None:
logging.error('Need a config file to perform such operation! Use --conf conf_file')
else:
batch_size = args.batch_size[0] if args.batch_size else 200
es = cls.new_instance(config) if not es else es
try:
if args.verbose > 2:
es.activate_debug()
if args.keys:
es.index_all_reviews(args.keys, replace=args.replace, batch_size=batch_size)
elif args.all_keys:
es.index_all_reviews(replace=args.replace, batch_size=batch_size)
except InvalidJsonResponseError as e:
logging.error('Invalid response! Reason: %s' % e)
sys.exit(1)
except IndexAlreadyExistsError:
logging.error('Index %s already exists! Use --recreate (with caution) to recreate' % index)
except ConnectionError:
logging.error('Could not connect to server at %s:%s' % (host, port))
except KeyError:
logging.error('Could not get host nor port! Use either -conf or --server')
sys.exit(1)
class ES:
url = None
index = None
es = None
def __init__(self, index=settings.DB_NAME):
self.es = ESConnection()
self.es.httprequest_kwargs['headers'] = {'Connection': 'keep-alive'}
self.index = index
@gen.coroutine
def search(self, mapping, query, extra_params):
fields = settings.RELAY_ES_FIELDS
return_data = list()
if query:
query = " AND ".join(query.split())
else:
query = "*"
time_q = extra_params['time']
query += self.__parse_extra(extra_params)
if mapping == "bridges":
fields = settings.BRIDGE_ES_FIELDS
body_query = {"_source": fields,
"query": {
"filtered": {
"query": {
"query_string": {
"query": query
}
},
"filter":{
"range": {
time_q['type']: {
"gte": time_q['range'][0] + " 00:00:00",
"lte": time_q['range'][1] + " 23:59:59"
}
}
}
}
}}
if settings.DEBUG:
print body_query
result = yield self.es.search(index=self.index, type=mapping, source=body_query, size=settings.ES_RESULT_SIZE)
result = escape.json_decode(result.body)
if 'hits' in result and 'hits' in result['hits']:
for hit in result['hits']['hits']:
return_data.append(hit['_source'])
else:
raise Exception(result['error'])
raise gen.Return(return_data)
def __parse_extra(self, extra):
query = ""
if 'country' in extra:
new_country = list()
geo_split = extra['country'].split(',')
for geo in geo_split:
new_country.append("geo:" + geo)
query = "{0} AND (".format(query) + " OR ".join(new_country) + ")"
if 'flags' in extra:
flags_split = extra['flags'].split(',')
for flag in flags_split:
query = "{0} AND flags:{1}".format(query, flag.title())
return query
class TestESConnection(AsyncTestCase):
def setUp(self):
self.io_loop = self.get_new_ioloop()
self.es_connection = ESConnection("localhost", "9200", self.io_loop)
def tearDown(self):
if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
self.io_loop.close(all_fds=True)
super(AsyncTestCase, self).tearDown()
def test_simple_search(self):
self.es_connection.get_by_path("/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'http://localhost/noticia/2/fast')
def test_search_for_specific_type_with_query(self):
self.es_connection.search(callback=self.stop,
source={"query": {"text": {"ID": "171171"}}},
type="materia", index="teste")
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')
def test_search_all_entries(self):
self.es_connection.search(self.stop)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 28)
def test_search_specific_index(self):
self.es_connection.search(callback=self.stop, index="outroteste")
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 14)
def test_search_apecific_type(self):
self.es_connection.search(self.stop, type='galeria')
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 2)
def test_should_access_specific_documento(self):
self.es_connection.get(index="teste", type="materia", uid="171171", callback=self.stop)
response = self.wait()
self.assertEqual(response['Portal'], "G1")
self.assertEqual(response['Macrotema'], "Noticias")
def test_should_accumulate_searches_before_search(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search("teste", source=source)
source = {"query": {"text": {"body": "multisearch"}}}
self.es_connection.multi_search("neverEndIndex", source=source)
self.assertListEqual(['{"index": "teste"}\n{"query": {"text": {"_id": "171171"}}}',
'{"index": "neverEndIndex"}\n{"query": {"text": {"body": "multisearch"}}}'
], self.es_connection.bulk.bulk_list)
def test_should_generate_empty_header_with_no_index_specified(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search(index=None, source=source)
source = {"query": {"text": {"body": "multisearch"}}}
self.es_connection.multi_search(index=None, source=source)
self.assertListEqual(['{}\n{"query": {"text": {"_id": "171171"}}}',
'{}\n{"query": {"text": {"body": "multisearch"}}}'
], self.es_connection.bulk.bulk_list)
def test_should_make_two_searches(self):
self._make_multisearch()
response = self._verify_status_code_and_return_response()
self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'], "171171")
self.assertFalse("hits" in response['responses'][1])
def test_should_clean_search_list_after_search(self):
self._make_multisearch()
self.wait()
self.assertListEqual([], self.es_connection.bulk.bulk_list)
def _make_multisearch(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search(index="teste", source=source)
source = {"query": {"text": {"_id": "101010"}}}
self.es_connection.multi_search(index="neverEndIndex", source=source)
self.es_connection.apply_search(callback=self.stop)
def _verify_status_code_and_return_response(self):
response = self.wait()
self.assertEqual(response.code, 200, "Wrong response code.")
response = escape.json_decode(response.body)
return response
class TestESConnectionWithTornadoGen(AsyncTestCase):
def setUp(self):
self.io_loop = self.get_new_ioloop()
self.es_connection = ESConnection("localhost", "9200", self.io_loop)
def tearDown(self):
if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
self.io_loop.close(all_fds=True)
super(AsyncTestCase, self).tearDown()
@gen_test
def test_simple_search(self):
response = yield self.es_connection.get_by_path(
"/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"],
u'http://localhost/noticia/2/fast')
@gen_test
def test_search_for_specific_type_with_query(self):
response = yield self.es_connection.search(
source={"query": {
"term": {
"ID": "171171"
}
}},
type="materia",
index="teste")
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')
@gen_test
def test_search_specific_index(self):
response = yield self.es_connection.search(index="outroteste")
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 14)
@gen_test
def test_search_apecific_type(self):
response = yield self.es_connection.search(type='galeria')
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response["hits"]["total"], 2)
@gen_test
def test_should_access_specific_documento(self):
response = yield self.es_connection.get(index="teste",
type="materia",
uid="171171")
self.assertEqual(response['Portal'], "G1")
self.assertEqual(response['Macrotema'], "Noticias")
@gen_test
def test_should_make_two_searches(self):
self._make_multisearch()
response = yield self.es_connection.apply_search()
response = self._verify_status_code_and_return_response(response)
self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'],
"171171")
self.assertFalse("hits" in response['responses'][1])
@gen_test
def test_should_clean_search_list_after_search(self):
self._make_multisearch()
response = yield self.es_connection.apply_search()
response = self._verify_status_code_and_return_response(response)
self.assertListEqual([], self.es_connection.bulk.bulk_list)
@gen_test
def test_can_put_and_delete_document(self):
try:
doc_id = str(uuid4())
response = yield self.es_connection.put(
"test",
"document",
doc_id, {
"test": "document",
"other": "property"
},
parameters={'refresh': True})
response_dict = self._verify_status_code_and_return_response(
response)
self.assertEqual(response_dict['_index'], 'test')
self.assertEqual(response_dict['_type'], 'document')
self.assertEqual(response_dict['_id'], doc_id)
self.assertIn('refresh=True', response.request.url)
finally:
response = yield self.es_connection.delete(
"test", "document", doc_id, parameters={'refresh': True})
response = self._verify_status_code_and_return_response(response)
self.assertTrue(response['found'])
self.assertEqual(response['_index'], 'test')
self.assertEqual(response['_type'], 'document')
self.assertEqual(response['_id'], doc_id)
@gen_test
def test_count_specific_index(self):
response = yield self.es_connection.count(index="outroteste")
self.assertCount(response, 14)
@gen_test
def test_count_specific_type(self):
response = yield self.es_connection.count(type='galeria')
self.assertCount(response, 2)
@gen_test
def test_count_specific_query(self):
source = {"query": {"term": {"_id": "171171"}}}
response = yield self.es_connection.count(source=source)
self.assertCount(response, 1)
@gen_test
def test_count_specific_query_with_parameters(self):
source = {"query": {"term": {"_id": "171171"}}}
parameters = {'refresh': True}
response = yield self.es_connection.count(callback=self.stop,
source=source,
parameters=parameters)
self.assertCount(response, 1)
self.assertTrue(response.request.url.endswith('_count?refresh=True'))
@gen_test
def test_count_specific_query_with_many_parameters(self):
source = {"query": {"term": {"_id": "171171"}}}
parameters = {'df': '_id', 'test': True}
response = yield self.es_connection.count(callback=self.stop,
source=source,
parameters=parameters)
self.assertCount(response, 1)
self.assertTrue(
response.request.url.endswith('_count?df=_id&test=True'))
def assertCount(self, response, count):
response_dict = self._verify_status_code_and_return_response(response)
self.assertEqual(response_dict["count"], count)
def _make_multisearch(self):
source = {"query": {"term": {"_id": "171171"}}}
self.es_connection.multi_search(index="teste", source=source)
source = {"query": {"term": {"_id": "101010"}}}
self.es_connection.multi_search(index="neverEndIndex", source=source)
def _verify_status_code_and_return_response(self, response):
self.assertTrue(response.code in [200, 201],
"Wrong response code: %d." % response.code)
response = escape.json_decode(response.body)
return response
class TestESConnection(AsyncTestCase):
def setUp(self):
self.io_loop = self.get_new_ioloop()
self.es_connection = ESConnection("localhost", "9200", self.io_loop)
def tearDown(self):
if (not IOLoop.initialized() or self.io_loop is not IOLoop.instance()):
self.io_loop.close(all_fds=True)
super(AsyncTestCase, self).tearDown()
def test_simple_search(self):
self.es_connection.get_by_path("/_search?q=_id:http\:\/\/localhost\/noticia\/2\/fast", self.stop)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'http://localhost/noticia/2/fast')
def test_search_for_specific_type_with_query(self):
self.es_connection.search(callback=self.stop,
source={"query": {"text": {"ID": "171171"}}},
type="materia", index="teste")
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 1)
self.assertEqual(response["hits"]["hits"][0]["_id"], u'171171')
def test_search_all_entries(self):
self.es_connection.search(self.stop)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 28)
def test_search_specific_index(self):
self.es_connection.search(callback=self.stop, index="outroteste")
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 14)
def test_search_apecific_type(self):
self.es_connection.search(self.stop, type='galeria')
response = self._verify_status_code_and_return_response()
self.assertEqual(response["hits"]["total"], 2)
def test_should_access_specific_documento(self):
self.es_connection.get(index="teste", type="materia", uid="171171", callback=self.stop)
response = self.wait()
self.assertEqual(response['Portal'], "G1")
self.assertEqual(response['Macrotema'], "Noticias")
def test_should_accumulate_searches_before_search(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search("teste", source=source)
source = {"query": {"text": {"body": "multisearch"}}}
self.es_connection.multi_search("neverEndIndex", source=source)
self.assertListEqual(['{"index": "teste"}\n{"query": {"text": {"_id": "171171"}}}',
'{"index": "neverEndIndex"}\n{"query": {"text": {"body": "multisearch"}}}'
], self.es_connection.bulk.bulk_list)
def test_should_generate_empty_header_with_no_index_specified(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search(index=None, source=source)
source = {"query": {"text": {"body": "multisearch"}}}
self.es_connection.multi_search(index=None, source=source)
self.assertListEqual(['{}\n{"query": {"text": {"_id": "171171"}}}',
'{}\n{"query": {"text": {"body": "multisearch"}}}'
], self.es_connection.bulk.bulk_list)
def test_should_make_two_searches(self):
self._make_multisearch()
response = self._verify_status_code_and_return_response()
self.assertEqual(response['responses'][0]['hits']['hits'][0]['_id'], "171171")
self.assertFalse("hits" in response['responses'][1])
def test_should_clean_search_list_after_search(self):
self._make_multisearch()
self.wait()
self.assertListEqual([], self.es_connection.bulk.bulk_list)
def _make_multisearch(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.multi_search(index="teste", source=source)
source = {"query": {"text": {"_id": "101010"}}}
self.es_connection.multi_search(index="neverEndIndex", source=source)
self.es_connection.apply_search(callback=self.stop)
def _verify_status_code_and_return_response(self):
response = self.wait()
return self._verify_response_and_returns_dict(response)
def _verify_response_and_returns_dict(self, response):
self.assertTrue(response.code in [200, 201], "Wrong response code: %d." % response.code)
response = escape.json_decode(response.body)
return response
def test_can_put_and_delete_document(self):
try:
doc_id = str(uuid4())
self.es_connection.put("test", "document", doc_id, {
"test": "document",
"other": "property"
}, parameters={'refresh': True}, callback=self.stop)
response = self.wait()
response_dict = self._verify_response_and_returns_dict(response)
self.assertEqual(response_dict['_index'], 'test')
self.assertEqual(response_dict['_type'], 'document')
self.assertTrue(response_dict['ok'])
self.assertEqual(response_dict['_id'], doc_id)
self.assertIn('refresh=True', response.request.url)
finally:
self.es_connection.delete("test", "document", doc_id,
parameters={'refresh': True}, callback=self.stop)
response = self._verify_status_code_and_return_response()
self.assertTrue(response['found'])
self.assertTrue(response['ok'])
self.assertEqual(response['_index'], 'test')
self.assertEqual(response['_type'], 'document')
self.assertEqual(response['_id'], doc_id)
def test_count_all_entries(self):
self.es_connection.count(callback=self.stop)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["count"], 28)
def test_count_specific_index(self):
self.es_connection.count(callback=self.stop, index="outroteste")
response = self._verify_status_code_and_return_response()
self.assertEqual(response["count"], 14)
def test_count_specific_type(self):
self.es_connection.count(callback=self.stop, type='galeria')
response = self._verify_status_code_and_return_response()
self.assertEqual(response["count"], 2)
def test_count_specific_query(self):
source = {"query": {"text": {"_id": "171171"}}}
self.es_connection.count(callback=self.stop, source=source)
response = self._verify_status_code_and_return_response()
self.assertEqual(response["count"], 1)
def test_count_specific_query_with_parameters(self):
source = {"query": {"text": {"_id": "171171"}}}
parameters = {'refresh': True}
self.es_connection.count(callback=self.stop, source=source, parameters=parameters)
response = self.wait()
response_dict = self._verify_response_and_returns_dict(response)
self.assertEqual(response_dict["count"], 1)
self.assertTrue(response.request.url.endswith('_count?refresh=True'))
def test_count_specific_query_with_many_parameters(self):
source = {"query": {"text": {"_id": "171171"}}}
parameters = {'df': '_id', 'test': True}
self.es_connection.count(callback=self.stop, source=source, parameters=parameters)
response = self.wait()
response_dict = self._verify_response_and_returns_dict(response)
self.assertEqual(response_dict["count"], 1)
self.assertTrue(response.request.url.endswith('_count?df=_id&test=True'))