def test_list_of_indices(self, mocked_get_indexes): """Test that unexisting indices are handled correctly. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) mocked_get_indexes.return_value = ['socorro_unknown'] res = api.get() res_expected = { 'hits': [], 'total': 0, 'facets': {}, } self.assertEqual(res, res_expected) mocked_get_indexes.return_value = [ 'socorro_integration_test', 'something_that_does_not_exist', 'another_one' ] res = api.get() self.assertTrue('total' in res) self.assertEqual(res['total'], 21) self.assertTrue('hits' in res) self.assertEqual(len(res['hits']), res['total']) self.assertTrue('facets' in res) self.assertTrue('signature' in res['facets'])
def test_get_with_pagination(self): """Test a search with pagination returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) args = { '_results_number': '10', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 10) args = { '_results_number': '10', '_results_offset': '10', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 10) args = { '_results_number': '10', '_results_offset': '15', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 6) args = { '_results_number': '10', '_results_offset': '30', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 0)
def test_get_with_range_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test date now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(days=7) lastmonth = lastweek - datetime.timedelta(weeks=4) args = {"date": ["<%s" % lastweek, ">=%s" % lastmonth]} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_little_signature") # Test build id args = {"build_id": "<1234567890"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertTrue(res["hits"][0]["build_id"] < 1234567890) args = {"build_id": ">1234567889"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue(report["build_id"] > 1234567889) args = {"build_id": "<=1234567890"} res = api.get(**args) self.assertEqual(res["total"], 21) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue(report["build_id"] <= 1234567890)
def test_get_with_pagination(self): """Test a search with pagination returns expected results. """ with _get_config_manager().context() as config: api = SuperSearch(config) args = { '_results_number': '10', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 10) args = { '_results_number': '10', '_results_offset': '10', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 10) args = { '_results_number': '10', '_results_offset': '15', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 6) args = { '_results_number': '10', '_results_offset': '30', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertEqual(len(res['hits']), 0)
def test_get(self): """Test a search with default values returns the right structure. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) res = api.get() self.assertTrue('total' in res) self.assertEqual(res['total'], 21) self.assertTrue('hits' in res) self.assertEqual(len(res['hits']), res['total']) self.assertTrue('facets' in res) self.assertTrue('signature' in res['facets']) expected_signatures = [ { 'term': 'js::break_your_browser', 'count': 20 }, { 'term': 'my_bad', 'count': 1 }, ] self.assertEqual(res['facets']['signature'], expected_signatures) # Test fields are being renamed self.assertTrue('date' in res['hits'][0]) # date_processed > date self.assertTrue('build_id' in res['hits'][0]) # build > build_id self.assertTrue('platform' in res['hits'][0]) # os_name > platform
def test_get(self): """Test a search with default values returns the right structure. """ with _get_config_manager().context() as config: api = SuperSearch(config) res = api.get() self.assertTrue('total' in res) self.assertEqual(res['total'], 21) self.assertTrue('hits' in res) self.assertEqual(len(res['hits']), res['total']) self.assertTrue('facets' in res) self.assertTrue('signature' in res['facets']) expected_signatures = [ {'term': 'js::break_your_browser', 'count': 20}, {'term': 'my_bad', 'count': 1}, ] self.assertEqual(res['facets']['signature'], expected_signatures) # Test fields are being renamed self.assertTrue('date' in res['hits'][0]) # date_processed > date self.assertTrue('build_id' in res['hits'][0]) # build > build_id self.assertTrue('platform' in res['hits'][0]) # os_name > platform
def test_get_with_range_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test date now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(days=7) lastmonth = lastweek - datetime.timedelta(weeks=4) args = { 'date': [ '<%s' % lastweek, '>=%s' % lastmonth, ] } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_little_signature') # Test build id args = { 'build_id': '<1234567890', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue(res['hits'][0]['build_id'] < 1234567890) args = { 'build_id': '>1234567889', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build_id'] > 1234567889) args = { 'build_id': '<=1234567890', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build_id'] <= 1234567890)
def test_get_with_range_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager().context() as config: api = SuperSearch(config) # Test date now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(days=7) lastmonth = lastweek - datetime.timedelta(weeks=4) args = { 'date': [ '<%s' % lastweek, '>=%s' % lastmonth, ] } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_little_signature') # Test build id args = { 'build_id': '<1234567890', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue(res['hits'][0]['build'] < 1234567890) args = { 'build_id': '>1234567889', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build'] > 1234567889) args = { 'build_id': '<=1234567890', } res = api.get(**args) self.assertEqual(res['total'], 21) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build'] <= 1234567890)
def test_get_with_not_operator(self): """Test a search with a few NOT operators. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = {"signature": ["js", "break_your_browser"]} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") # - Test contains mode args = {"signature": "!~bad"} res = api.get(**args) self.assertEqual(res["total"], 20) # - Test is_exactly mode args = {"signature": "!=js::break_your_browser"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") # - Test starts_with mode args = {"signature": "!$js"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") # - Test ends_with mode args = {"signature": "!^browser"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") # Test build id args = {"build_id": "!<1234567890"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue(report["build_id"] > 1234567889) args = {"build_id": "!>1234567889"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertTrue(res["hits"][0]["build_id"] < 1234567890) args = {"build_id": "!<=1234567890"} res = api.get(**args) self.assertEqual(res["total"], 0)
def test_get_with_facets(self): """Test a search with facets returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test several facets args = {"_facets": ["signature", "platform"]} res = api.get(**args) self.assertTrue("facets" in res) self.assertTrue("signature" in res["facets"]) expected_signatures = [{"term": "js::break_your_browser", "count": 20}, {"term": "my_bad", "count": 1}] self.assertEqual(res["facets"]["signature"], expected_signatures) self.assertTrue("platform" in res["facets"]) expected_platforms = [{"term": "Linux", "count": 20}, {"term": "Windows NT", "count": 1}] self.assertEqual(res["facets"]["platform"], expected_platforms) # Test one facet with filters args = {"_facets": ["release_channel"], "release_channel": "aurora"} res = api.get(**args) self.assertTrue("release_channel" in res["facets"]) expected_signatures = [{"term": "aurora", "count": 1}] self.assertEqual(res["facets"]["release_channel"], expected_signatures) # Test one facet with a different filter args = {"_facets": ["release_channel"], "platform": "linux"} res = api.get(**args) self.assertTrue("release_channel" in res["facets"]) expected_signatures = [{"term": "release", "count": 19}, {"term": "aurora", "count": 1}] self.assertEqual(res["facets"]["release_channel"], expected_signatures) # Test errors self.assertRaises(BadArgumentError, api.get, _facets=["unkownfield"])
def test_get_with_pagination(self): """Test a search with pagination returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) args = {"_results_number": "10"} res = api.get(**args) self.assertEqual(res["total"], 21) self.assertEqual(len(res["hits"]), 10) args = {"_results_number": "10", "_results_offset": "10"} res = api.get(**args) self.assertEqual(res["total"], 21) self.assertEqual(len(res["hits"]), 10) args = {"_results_number": "10", "_results_offset": "15"} res = api.get(**args) self.assertEqual(res["total"], 21) self.assertEqual(len(res["hits"]), 6) args = {"_results_number": "10", "_results_offset": "30"} res = api.get(**args) self.assertEqual(res["total"], 21) self.assertEqual(len(res["hits"]), 0)
def test_get(self): """Test a search with default values returns the right structure. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) res = api.get() self.assertTrue("total" in res) self.assertEqual(res["total"], 21) self.assertTrue("hits" in res) self.assertEqual(len(res["hits"]), res["total"]) self.assertTrue("facets" in res) self.assertTrue("signature" in res["facets"]) expected_signatures = [{"term": "js::break_your_browser", "count": 20}, {"term": "my_bad", "count": 1}] self.assertEqual(res["facets"]["signature"], expected_signatures) # Test fields are being renamed self.assertTrue("date" in res["hits"][0]) # date_processed > date self.assertTrue("build_id" in res["hits"][0]) # build > build_id self.assertTrue("platform" in res["hits"][0]) # os_name > platform
def test_get_with_facets(self): """Test a search with facets returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test several facets args = {'_facets': ['signature', 'platform']} res = api.get(**args) self.assertTrue('facets' in res) self.assertTrue('signature' in res['facets']) expected_signatures = [ { 'term': 'js::break_your_browser', 'count': 20 }, { 'term': 'my_bad', 'count': 1 }, ] self.assertEqual(res['facets']['signature'], expected_signatures) self.assertTrue('platform' in res['facets']) expected_platforms = [ { 'term': 'Linux', 'count': 20 }, { 'term': 'Windows NT', 'count': 1 }, ] self.assertEqual(res['facets']['platform'], expected_platforms) # Test one facet with filters args = { '_facets': ['release_channel'], 'release_channel': 'aurora', } res = api.get(**args) self.assertTrue('release_channel' in res['facets']) expected_signatures = [ { 'term': 'aurora', 'count': 1 }, ] self.assertEqual(res['facets']['release_channel'], expected_signatures) # Test one facet with a different filter args = { '_facets': ['release_channel'], 'platform': 'linux', } res = api.get(**args) self.assertTrue('release_channel' in res['facets']) expected_signatures = [ { 'term': 'release', 'count': 19 }, { 'term': 'aurora', 'count': 1 }, ] self.assertEqual(res['facets']['release_channel'], expected_signatures) # Test errors self.assertRaises(BadArgumentError, api.get, _facets=['unkownfield'])
def test_get_with_string_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = {"signature": ["js", "break_your_browser"]} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") # - Test contains mode args = {"signature": "~bad"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") args = {"signature": "~js::break"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") # - Test is_exactly mode args = {"signature": "=js::break_your_browser"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") args = {"signature": "=my_bad"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") # - Test starts_with mode args = {"signature": "$js"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") # - Test ends_with mode args = {"signature": "^browser"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertEqual(report["signature"], "js::break_your_browser") # Test email args = {"email": "*****@*****.**"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertTrue(res["hits"]) self.assertEqual(res["hits"][0]["email"], "*****@*****.**") args = {"email": "~mail.com"} res = api.get(**args) self.assertEqual(res["total"], 19) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue("@" in report["email"]) self.assertTrue("mail.com" in report["email"]) args = {"email": "$sauron@"} res = api.get(**args) self.assertEqual(res["total"], 2) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue("sauron@" in report["email"]) # Test url args = {"url": "https://mozilla.org"} res = api.get(**args) self.assertEqual(res["total"], 19) args = {"url": "~mozilla.org"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue(res["hits"]) for report in res["hits"]: self.assertTrue("mozilla.org" in report["url"]) args = {"url": "^.com"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["url"], "http://www.example.com") # Test user_comments args = {"user_comments": "~love"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["user_comments"], "I love WaterWolf") args = {"user_comments": "$WaterWolf"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["user_comments"], "WaterWolf is so bad") args = {"user_comments": "__null__"} res = api.get(**args) self.assertEqual(res["total"], 19) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") for hit in res["hits"]: self.assertEqual(hit["user_comments"], "") # Test address args = {"address": "^0"} res = api.get(**args) self.assertEqual(res["total"], 21) args = {"address": "~a2"} res = api.get(**args) self.assertEqual(res["total"], 1)
def test_get_with_string_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = { 'signature': ['js', 'break_your_browser'], } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test contains mode args = { 'signature': '~bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') args = { 'signature': '~js::break', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test is_exactly mode args = { 'signature': '=js::break_your_browser', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') args = { 'signature': '=my_bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode args = { 'signature': '$js', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test ends_with mode args = { 'signature': '^browser', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # Test email args = { 'email': '*****@*****.**', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertTrue(res['hits']) self.assertEqual(res['hits'][0]['email'], '*****@*****.**') args = { 'email': '~mail.com', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('@' in report['email']) self.assertTrue('mail.com' in report['email']) args = { 'email': '$sauron@', } res = api.get(**args) self.assertEqual(res['total'], 2) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('sauron@' in report['email']) # Test url args = { 'url': 'https://mozilla.org', } res = api.get(**args) self.assertEqual(res['total'], 19) args = { 'url': '~mozilla.org', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('mozilla.org' in report['url']) args = { 'url': '^.com', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['url'], 'http://www.example.com') # Test user_comments args = { 'user_comments': '~love', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['user_comments'], 'I love WaterWolf') args = { 'user_comments': '$WaterWolf', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['user_comments'], 'WaterWolf is so bad') args = { 'user_comments': '__null__', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') for hit in res['hits']: self.assertEqual(hit['user_comments'], '') # Test address args = { 'address': '^0', } res = api.get(**args) self.assertEqual(res['total'], 21) args = { 'address': '~a2', } res = api.get(**args) self.assertEqual(res['total'], 1)
class IntegrationTestSettings(ElasticSearchTestCase): """Test the settings and mappings used in elasticsearch, through the supersearch service. """ def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) self.api = SuperSearch(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) # This an ugly hack to give elasticsearch some time to finish creating # the new index. It is needed for jenkins only, because we have a # special case here where we index only one or two documents before # querying. Other tests are not affected. # TODO: try to remove it, or at least understand why it is needed. time.sleep(1) def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) super(IntegrationTestSettings, self).tearDown() def test_dump_field(self): """Verify that the 'dump' field can be queried as expected. """ # Index some data. processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120100", "date_processed": self.now, "dump": EXAMPLE_DUMP, } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(dump="~family") self.assertEqual(res["total"], 1) # Several words, with upper case. res = self.api.get(dump="~Windows NT") self.assertEqual(res["total"], 1) def test_cpu_info_field(self): """Verify that the 'cpu_info' field can be queried as expected. """ processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120101", "date_processed": self.now, "cpu_info": "GenuineIntel family 6 model 15 stepping 13", } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(cpu_info="~model") self.assertEqual(res["total"], 1) self.assertTrue("model" in res["hits"][0]["cpu_info"]) # Several words, with upper case, 'starts with' mode. res = self.api.get(cpu_info="$GenuineIntel family") self.assertEqual(res["total"], 1) self.assertTrue("GenuineIntel family" in res["hits"][0]["cpu_info"]) def test_dom_ipc_enabled_field(self): """Verify that the 'dom_ipc_enabled' field can be queried as expected. """ processed_crash = {"uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120101", "date_processed": self.now} raw_crash = {"DOMIPCEnabled": True} self.storage.save_raw_and_processed(raw_crash, None, processed_crash, processed_crash["uuid"]) processed_crash = {"uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120102", "date_processed": self.now} raw_crash = {"DOMIPCEnabled": False} self.storage.save_raw_and_processed(raw_crash, None, processed_crash, processed_crash["uuid"]) processed_crash = {"uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120103", "date_processed": self.now} raw_crash = {"DOMIPCEnabled": None} self.storage.save_raw_and_processed(raw_crash, None, processed_crash, processed_crash["uuid"]) self.storage.es.refresh() res = self.api.get(dom_ipc_enabled="true") self.assertEqual(res["total"], 1) self.assertTrue(res["hits"][0]["dom_ipc_enabled"]) res = self.api.get(dom_ipc_enabled="false") self.assertEqual(res["total"], 2) def test_platform_field(self): """Verify that the 'platform' field can be queried as expected. """ processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120102", "date_processed": self.now, "os_name": "Mac OS X", } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(platform="Mac OS X") self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["platform"], "Mac OS X") def test_app_notes_field(self): """Verify that the 'app_notes' field can be queried as expected. """ processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120103", "date_processed": self.now, "app_notes": "there is a cycle collector fault here", } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(app_notes="cycle collector fault") self.assertEqual(res["total"], 1) self.assertTrue("cycle collector fault" in res["hits"][0]["app_notes"]) def test_process_type_field(self): """Verify that the 'process_type' field can be queried as expected. """ processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120100", "date_processed": self.now, "process_type": "plugin", } self.storage.save_processed(processed_crash) processed_crash = { "uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120101", "date_processed": self.now, "process_type": None, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(process_type="plugin") self.assertEqual(res["total"], 1) self.assertTrue("plugin" in res["hits"][0]["process_type"]) res = self.api.get(process_type="browser") self.assertEqual(res["total"], 1) # In the case of a 'browser' crash, the process_type is None and thus # is not returned. self.assertTrue("process_type" not in res["hits"][0]) res = self.api.get(process_type="!browser") self.assertEqual(res["total"], 1) self.assertTrue("plugin" in res["hits"][0]["process_type"]) res = self.api.get(process_type=["plugin", "browser"]) self.assertEqual(res["total"], 2) def test_hang_type_field(self): """Verify that the 'hang_type' field can be queried as expected. """ processed_crash = {"uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120100", "date_processed": self.now, "hang_type": 0} self.storage.save_processed(processed_crash) processed_crash = {"uuid": "06a0c9b5-0381-42ce-855a-ccaaa2120101", "date_processed": self.now, "hang_type": 1} self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(hang_type="hang") self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["hang_type"], 1) res = self.api.get(hang_type="crash") self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["hang_type"], 0) res = self.api.get(hang_type="!crash") self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["hang_type"], 1) res = self.api.get(hang_type=["crash", "hang"]) self.assertEqual(res["total"], 2)
def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = { 'signature': 'my_bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # Test product args = { 'product': 'EarthRaccoon', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['product'], 'EarthRaccoon') # Test version args = { 'version': '2.0', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['version'], '2.0') # Test release_channel args = { 'release_channel': 'aurora', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['release_channel'], 'aurora') # Test platform args = { 'platform': 'Windows', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['platform'], 'Windows NT') # Test build_id args = { 'build_id': '987654321', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['build_id'], 987654321) # Test reason args = { 'reason': 'MOZALLOC_WENT_WRONG', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertEqual(res['hits'][0]['reason'], 'MOZALLOC_WENT_WRONG') args = { 'reason': ['very_bad_exception'], } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['reason'], 'VERY_BAD_EXCEPTION') # Test process_type args = { 'process_type': 'plugin', } res = api.get(**args) self.assertEqual(res['total'], 3) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['process_type'], 'plugin') # Test url args = { 'url': 'https://mozilla.org', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('mozilla.org' in res['hits'][0]['url']) # Test user_comments args = { 'user_comments': 'WaterWolf', } res = api.get(**args) self.assertEqual(res['total'], 2) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('WaterWolf' in res['hits'][0]['user_comments']) # Test address args = { 'address': '0x0', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue('0x0' in res['hits'][0]['address'])
class IntegrationTestSettings(ElasticSearchTestCase): """Test the settings and mappings used in elasticsearch, through the supersearch service. """ def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) self.api = SuperSearch(config=config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) # This an ugly hack to give elasticsearch some time to finish creating # the new index. It is needed for jenkins only, because we have a # special case here where we index only one or two documents before # querying. Other tests are not affected. # TODO: try to remove it, or at least understand why it is needed. time.sleep(1) def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) super(IntegrationTestSettings, self).tearDown() def test_dump_field(self): """Verify that the 'dump' field can be queried as expected. """ # Index some data. processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'dump': EXAMPLE_DUMP, } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(dump='~family') eq_(res['total'], 1) # Several words, with upper case. res = self.api.get(dump='~Windows NT') eq_(res['total'], 1) def test_cpu_info_field(self): """Verify that the 'cpu_info' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'cpu_info': 'GenuineIntel family 6 model 15 stepping 13', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(cpu_info='~model') eq_(res['total'], 1) ok_('model' in res['hits'][0]['cpu_info']) # Several words, with upper case, 'starts with' mode. res = self.api.get(cpu_info='$GenuineIntel family') eq_(res['total'], 1) ok_('GenuineIntel family' in res['hits'][0]['cpu_info']) def test_dom_ipc_enabled_field(self): """Verify that the 'dom_ipc_enabled' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': True, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': False, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': None, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) self.storage.es.refresh() res = self.api.get(dom_ipc_enabled='true') eq_(res['total'], 1) ok_(res['hits'][0]['dom_ipc_enabled']) res = self.api.get(dom_ipc_enabled='false') eq_(res['total'], 2) def test_platform_field(self): """Verify that the 'platform' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, 'os_name': 'Mac OS X', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(platform='Mac OS X') eq_(res['total'], 1) eq_(res['hits'][0]['platform'], 'Mac OS X') def test_app_notes_field(self): """Verify that the 'app_notes' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, 'app_notes': 'there is a cycle collector fault here', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(app_notes='cycle collector fault') eq_(res['total'], 1) ok_('cycle collector fault' in res['hits'][0]['app_notes']) def test_process_type_field(self): """Verify that the 'process_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'process_type': 'plugin', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'process_type': None, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(process_type='plugin') eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type='browser') eq_(res['total'], 1) # In the case of a 'browser' crash, the process_type is None and thus # is not returned. ok_('process_type' not in res['hits'][0]) res = self.api.get(process_type='!browser') eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type=['plugin', 'browser']) eq_(res['total'], 2) def test_hang_type_field(self): """Verify that the 'hang_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'hang_type': 0, } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'hang_type': 1, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(hang_type='hang') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type='crash') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 0) res = self.api.get(hang_type='!crash') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type=['crash', 'hang']) eq_(res['total'], 2) def test_exploitability_field(self): """Verify that the 'exploitability' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'exploitability': 'high', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'exploitability': 'unknown', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(exploitability='high') eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'high') res = self.api.get(exploitability='unknown') eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'unknown') res = self.api.get(exploitability=['high', 'unknown']) eq_(res['total'], 2) def test_platform_version_field(self): """Verify that the 'platform_version' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'os_version': '6.0.001', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'os_version': '6.0.001 Service Pack 1', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(platform_version='6.0.001') eq_(res['total'], 2) ok_('6.0.001' in res['hits'][0]['platform_version']) ok_('6.0.001' in res['hits'][1]['platform_version']) res = self.api.get(platform_version='6.0.001 Service Pack 1') eq_(res['total'], 1) eq_(res['hits'][0]['platform_version'], '6.0.001 Service Pack 1') res = self.api.get(platform_version='$6.0') eq_(res['total'], 2) ok_('6.0' in res['hits'][0]['platform_version']) ok_('6.0' in res['hits'][1]['platform_version'])
class IntegrationTestSettings(ElasticSearchTestCase): """Test the settings and mappings used in elasticsearch, through the supersearch service. """ def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the supersearch fields. self.storage.es.bulk_index( index=config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', docs=SUPERSEARCH_FIELDS.values(), id_field='name', refresh=True, ) # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) self.api = SuperSearch(config=config) def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) self.storage.es.delete_index(config.webapi.elasticsearch_default_index) super(IntegrationTestSettings, self).tearDown() def test_dump_field(self): """Verify that the 'dump' field can be queried as expected. """ # Index some data. processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'dump': EXAMPLE_DUMP, } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(dump='~family') eq_(res['total'], 1) # Several words, with upper case. res = self.api.get(dump='~Windows NT') eq_(res['total'], 1) @maximum_es_version('0.90') def test_cpu_info_field(self): """Verify that the 'cpu_info' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'cpu_info': 'GenuineIntel family 6 model 15 stepping 13', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(cpu_info='~model') eq_(res['total'], 1) ok_('model' in res['hits'][0]['cpu_info']) # Several words, with upper case, 'starts with' mode. res = self.api.get(cpu_info='$GenuineIntel family') eq_(res['total'], 1) ok_('GenuineIntel family' in res['hits'][0]['cpu_info']) def test_dom_ipc_enabled_field(self): """Verify that the 'dom_ipc_enabled' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': True, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': False, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': None, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) self.storage.es.refresh() res = self.api.get(dom_ipc_enabled='true') eq_(res['total'], 1) ok_(res['hits'][0]['dom_ipc_enabled']) res = self.api.get(dom_ipc_enabled='false') eq_(res['total'], 2) @maximum_es_version('0.90') def test_platform_field(self): """Verify that the 'platform' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, 'os_name': 'Mac OS X', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(platform='Mac OS X') eq_(res['total'], 1) eq_(res['hits'][0]['platform'], 'Mac OS X') @maximum_es_version('0.90') def test_app_notes_field(self): """Verify that the 'app_notes' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, 'app_notes': 'there is a cycle collector fault here', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(app_notes='cycle collector fault') eq_(res['total'], 1) ok_('cycle collector fault' in res['hits'][0]['app_notes']) def test_process_type_field(self): """Verify that the 'process_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'process_type': 'plugin', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'process_type': None, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(process_type='plugin') eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type='browser') eq_(res['total'], 1) # In the case of a 'browser' crash, the process_type is None and thus # is not returned. ok_('process_type' not in res['hits'][0]) res = self.api.get(process_type='!browser') eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type=['plugin', 'browser']) eq_(res['total'], 2) @maximum_es_version('0.90') def test_hang_type_field(self): """Verify that the 'hang_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'hang_type': 0, } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'hang_type': 1, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(hang_type='hang') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type='crash') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 0) res = self.api.get(hang_type='!crash') eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type=['crash', 'hang']) eq_(res['total'], 2) @maximum_es_version('0.90') def test_exploitability_field(self): """Verify that the 'exploitability' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'exploitability': 'high', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'exploitability': 'unknown', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(exploitability='high') eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'high') res = self.api.get(exploitability='unknown') eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'unknown') res = self.api.get(exploitability=['high', 'unknown']) eq_(res['total'], 2) @maximum_es_version('0.90') def test_platform_version_field(self): """Verify that the 'platform_version' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'os_version': '6.0.001', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'os_version': '6.0.001 Service Pack 1', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(platform_version='6.0.001') eq_(res['total'], 2) ok_('6.0.001' in res['hits'][0]['platform_version']) ok_('6.0.001' in res['hits'][1]['platform_version']) res = self.api.get(platform_version='6.0.001 Service Pack 1') eq_(res['total'], 1) eq_(res['hits'][0]['platform_version'], '6.0.001 Service Pack 1') res = self.api.get(platform_version='$6.0') eq_(res['total'], 2) ok_('6.0' in res['hits'][0]['platform_version']) ok_('6.0' in res['hits'][1]['platform_version'])
class IntegrationTestSettings(ElasticSearchTestCase): """Test the settings and mappings used in elasticsearch, through the supersearch service. """ def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the supersearch fields. self.storage.es.bulk_index( index=config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', docs=SUPERSEARCH_FIELDS.values(), id_field='name', refresh=True, ) # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) self.api = SuperSearch(config=config) def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) self.storage.es.delete_index(config.webapi.elasticsearch_default_index) super(IntegrationTestSettings, self).tearDown() def test_dump_field(self): """Verify that the 'dump' field can be queried as expected. """ # Index some data. processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'dump': EXAMPLE_DUMP, } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(dump='~family') eq_(res['total'], 1) # Several words, with upper case. res = self.api.get(dump='~Windows NT') eq_(res['total'], 1) @maximum_es_version('0.90.99') def test_cpu_info_field(self): """Verify that the 'cpu_info' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'cpu_info': 'GenuineIntel family 6 model 15 stepping 13', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get( cpu_info='~model', _columns=['cpu_info'], ) eq_(res['total'], 1) ok_('model' in res['hits'][0]['cpu_info']) # Several words, with upper case, 'starts with' mode. res = self.api.get( cpu_info='$GenuineIntel family', _columns=['cpu_info'], ) eq_(res['total'], 1) ok_('GenuineIntel family' in res['hits'][0]['cpu_info']) def test_dom_ipc_enabled_field(self): """Verify that the 'dom_ipc_enabled' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': True, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': False, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': None, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) self.storage.es.refresh() res = self.api.get( dom_ipc_enabled='true', _columns=['uuid', 'dom_ipc_enabled'], ) eq_(res['total'], 1) ok_(res['hits'][0]['dom_ipc_enabled']) res = self.api.get( dom_ipc_enabled='false', _columns=['uuid', 'dom_ipc_enabled'], ) eq_(res['total'], 2) @maximum_es_version('0.90.99') def test_platform_field(self): """Verify that the 'platform' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, 'os_name': 'Mac OS X', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get( platform='Mac OS X', _columns=['platform'], ) eq_(res['total'], 1) eq_(res['hits'][0]['platform'], 'Mac OS X') @maximum_es_version('0.90.99') def test_app_notes_field(self): """Verify that the 'app_notes' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, 'app_notes': 'there is a cycle collector fault here', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get( app_notes='cycle collector fault', _columns=['app_notes'], ) eq_(res['total'], 1) ok_('cycle collector fault' in res['hits'][0]['app_notes']) def test_process_type_field(self): """Verify that the 'process_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'process_type': 'plugin', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'process_type': None, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get( process_type='plugin', _columns=['uuid', 'process_type'], ) eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get( process_type='browser', _columns=['uuid', 'process_type'], ) eq_(res['total'], 1) # In the case of a 'browser' crash, the process_type is None and thus # is not returned. ok_('process_type' not in res['hits'][0]) res = self.api.get( process_type='!browser', _columns=['uuid', 'process_type'], ) eq_(res['total'], 1) ok_('plugin' in res['hits'][0]['process_type']) res = self.api.get( process_type=['plugin', 'browser'], _columns=['uuid', 'process_type'], ) eq_(res['total'], 2) @maximum_es_version('0.90.99') def test_hang_type_field(self): """Verify that the 'hang_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'hang_type': 0, } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'hang_type': 1, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get( hang_type='hang', _columns=['hang_type'], ) eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get( hang_type='crash', _columns=['hang_type'], ) eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 0) res = self.api.get( hang_type='!crash', _columns=['hang_type'], ) eq_(res['total'], 1) eq_(res['hits'][0]['hang_type'], 1) res = self.api.get( hang_type=['crash', 'hang'], _columns=['hang_type'], ) eq_(res['total'], 2) @maximum_es_version('0.90.99') def test_exploitability_field(self): """Verify that the 'exploitability' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'exploitability': 'high', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'exploitability': 'unknown', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get( exploitability='high', _columns=['exploitability'], ) eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'high') res = self.api.get( exploitability='unknown', _columns=['exploitability'], ) eq_(res['total'], 1) eq_(res['hits'][0]['exploitability'], 'unknown') res = self.api.get( exploitability=['high', 'unknown'], _columns=['exploitability'], ) eq_(res['total'], 2) @maximum_es_version('0.90.99') def test_platform_version_field(self): """Verify that the 'platform_version' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'os_version': '6.0.001', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'os_version': '6.0.001 Service Pack 1', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get( platform_version='6.0.001', _columns=['platform_version'], ) eq_(res['total'], 2) ok_('6.0.001' in res['hits'][0]['platform_version']) ok_('6.0.001' in res['hits'][1]['platform_version']) res = self.api.get( platform_version='6.0.001 Service Pack 1', _columns=['platform_version'], ) eq_(res['total'], 1) eq_(res['hits'][0]['platform_version'], '6.0.001 Service Pack 1') res = self.api.get( platform_version='$6.0', _columns=['platform_version'], ) eq_(res['total'], 2) ok_('6.0' in res['hits'][0]['platform_version']) ok_('6.0' in res['hits'][1]['platform_version'])
class IntegrationTestSuperSearch(ElasticSearchTestCase): """Test SuperSearch with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearch, self).setUp() config = self.get_config_context() self.api = SuperSearch(config=config) self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() now = datetimeutil.utc_now() yesterday = now - datetime.timedelta(days=1) yesterday = datetimeutil.date_to_string(yesterday) last_month = now - datetime.timedelta(weeks=4) last_month = datetimeutil.date_to_string(last_month) # insert data into elasticsearch default_crash_report = { 'uuid': 100, 'address': '0x0', 'signature': 'js::break_your_browser', 'date_processed': yesterday, 'product': 'WaterWolf', 'version': '1.0', 'release_channel': 'release', 'os_name': 'Linux', 'build': 1234567890, 'reason': 'MOZALLOC_WENT_WRONG', 'hangid': None, 'process_type': None, 'email': '*****@*****.**', 'url': 'https://mozilla.org', 'user_comments': '', 'install_age': 0, } default_raw_crash_report = { 'Accessibility': True, 'AvailableVirtualMemory': 10211743, 'B2G_OS_Version': '1.1.203448', 'BIOS_Manufacturer': 'SUSA', 'IsGarbageCollecting': False, 'Vendor': 'mozilla', 'useragent_locale': 'en-US', } self.storage.save_raw_and_processed( default_raw_crash_report, None, default_crash_report, default_crash_report['uuid'] ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Accessibility=False), None, dict(default_crash_report, uuid=1, product='EarthRaccoon'), 1 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, AvailableVirtualMemory=0), None, dict(default_crash_report, uuid=2, version='2.0'), 2 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, B2G_OS_Version='1.3'), None, dict(default_crash_report, uuid=3, release_channel='aurora'), 3 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, BIOS_Manufacturer='aidivn'), None, dict(default_crash_report, uuid=4, os_name='Windows NT'), 4 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, IsGarbageCollecting=True), None, dict(default_crash_report, uuid=5, build=987654321), 5 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Vendor='gnusmas'), None, dict(default_crash_report, uuid=6, reason='VERY_BAD_EXCEPTION'), 6 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, useragent_locale='fr'), None, dict(default_crash_report, uuid=7, hangid=12), 7 ) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Android_Model='PediaMad 17 Heavy'), None, dict(default_crash_report, uuid=8, process_type='plugin'), 8 ) self.storage.save_processed( dict(default_crash_report, uuid=9, signature='my_bad') ) self.storage.save_processed( dict( default_crash_report, uuid=10, date_processed=last_month, signature='my_little_signature', ) ) # for plugin terms test self.storage.save_processed( dict( default_crash_report, uuid=11, product='PluginSoft', process_type='plugin', PluginFilename='carly.dll', PluginName='Hey I just met you', PluginVersion='1.2', ) ) self.storage.save_processed( dict( default_crash_report, uuid=12, product='PluginSoft', process_type='plugin', PluginFilename='hey.dll', PluginName='Hey Plugin', PluginVersion='10.7.0.2a', ) ) self.storage.save_processed( dict(default_crash_report, uuid=13, email='*****@*****.**') ) self.storage.save_processed( dict(default_crash_report, uuid=14, email='*****@*****.**') ) self.storage.save_processed( dict(default_crash_report, uuid=15, email='*****@*****.**') ) self.storage.save_processed( dict(default_crash_report, uuid=16, install_age=87234) ) self.storage.save_processed( dict(default_crash_report, uuid=17, url='http://www.mozilla.org') ) self.storage.save_processed( dict(default_crash_report, uuid=18, url='http://www.example.com') ) self.storage.save_processed( dict( default_crash_report, uuid=19, user_comments='I love WaterWolf', ) ) self.storage.save_processed( dict( default_crash_report, uuid=20, user_comments='WaterWolf is so bad', ) ) self.storage.save_processed( dict(default_crash_report, uuid=21, address='0xa2e4509ca0') ) # As indexing is asynchronous, we need to force elasticsearch to # make the newly created content searchable before we run the tests self.storage.es.refresh() def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) super(IntegrationTestSuperSearch, self).tearDown() def test_get(self): """Test a search with default values returns the right structure. """ res = self.api.get() ok_('total' in res) eq_(res['total'], 21) ok_('hits' in res) eq_(len(res['hits']), res['total']) ok_('facets' in res) ok_('signature' in res['facets']) expected_signatures = [ {'term': 'js::break_your_browser', 'count': 20}, {'term': 'my_bad', 'count': 1}, ] eq_(res['facets']['signature'], expected_signatures) # Test fields are being renamed ok_('date' in res['hits'][0]) # date_processed > date ok_('build_id' in res['hits'][0]) # build > build_id ok_('platform' in res['hits'][0]) # os_name > platform def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ # Test signature kwargs = { 'signature': 'my_bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # Test product kwargs = { 'product': 'EarthRaccoon', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['product'], 'EarthRaccoon') # Test version kwargs = { 'version': '2.0', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['version'], '2.0') # Test release_channel kwargs = { 'release_channel': 'aurora', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['release_channel'], 'aurora') # Test platform kwargs = { 'platform': 'Windows', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['platform'], 'Windows NT') # Test build_id kwargs = { 'build_id': '987654321', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['build_id'], 987654321) # Test reason kwargs = { 'reason': 'MOZALLOC_WENT_WRONG', } res = self.api.get(**kwargs) eq_(res['total'], 20) eq_(res['hits'][0]['reason'], 'MOZALLOC_WENT_WRONG') kwargs = { 'reason': ['very_bad_exception'], } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['reason'], 'VERY_BAD_EXCEPTION') # Test process_type kwargs = { 'process_type': 'plugin', } res = self.api.get(**kwargs) eq_(res['total'], 3) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['process_type'], 'plugin') # Test url kwargs = { 'url': 'https://mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 19) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_('mozilla.org' in res['hits'][0]['url']) # Test user_comments kwargs = { 'user_comments': 'WaterWolf', } res = self.api.get(**kwargs) eq_(res['total'], 2) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_('WaterWolf' in res['hits'][0]['user_comments']) # Test address kwargs = { 'address': '0x0', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_('0x0' in res['hits'][0]['address']) # Test accessibility kwargs = { 'accessibility': False, } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(not res['hits'][0]['accessibility']) kwargs = { 'accessibility': 'True', } res = self.api.get(**kwargs) eq_(res['total'], 8) ok_(res['hits'][0]['accessibility']) # Test b2g_os_version kwargs = { 'b2g_os_version': '1.3', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['b2g_os_version'], '1.3') # Test bios_manufacturer kwargs = { 'bios_manufacturer': 'aidivn', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['bios_manufacturer'], 'aidivn') # Test is_garbage_collecting kwargs = { 'is_garbage_collecting': True, } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(res['hits'][0]['is_garbage_collecting']) # Test vendor kwargs = { 'vendor': 'gnusmas', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['vendor'], 'gnusmas') # Test useragent_locale kwargs = { 'useragent_locale': 'fr', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['useragent_locale'], 'fr') def test_get_with_range_operators(self): """Test a search with several filters and operators returns expected results. """ # Test date now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(days=7) lastmonth = lastweek - datetime.timedelta(weeks=4) kwargs = { 'date': [ '<%s' % lastweek, '>=%s' % lastmonth, ] } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_little_signature') # Test build id kwargs = { 'build_id': '<1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_(res['hits'][0]['build_id'] < 1234567890) kwargs = { 'build_id': '>1234567889', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] > 1234567889) kwargs = { 'build_id': '<=1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 21) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] <= 1234567890) # Test available_virtual_memory kwargs = { 'available_virtual_memory': '>=1', } res = self.api.get(**kwargs) eq_(res['total'], 8) for report in res['hits']: ok_(report['available_virtual_memory'] >= 1) kwargs = { 'available_virtual_memory': '<1', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['available_virtual_memory'], 0) def test_get_with_string_operators(self): """Test a search with several filters and operators returns expected results. """ # Test signature kwargs = { 'signature': ['js', 'break_your_browser'], } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test contains mode kwargs = { 'signature': '~bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') kwargs = { 'signature': '~js::break', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test is_exactly mode kwargs = { 'signature': '=js::break_your_browser', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') kwargs = { 'signature': '=my_bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode kwargs = { 'signature': '$js', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test ends_with mode kwargs = { 'signature': '^browser', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # Test email kwargs = { 'email': '*****@*****.**', } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(res['hits']) eq_(res['hits'][0]['email'], '*****@*****.**') kwargs = { 'email': '~mail.com', } res = self.api.get(**kwargs) eq_(res['total'], 19) ok_(res['hits']) for report in res['hits']: ok_('@' in report['email']) ok_('mail.com' in report['email']) kwargs = { 'email': '$sauron@', } res = self.api.get(**kwargs) eq_(res['total'], 2) ok_(res['hits']) for report in res['hits']: ok_('sauron@' in report['email']) # Test url kwargs = { 'url': 'https://mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 19) kwargs = { 'url': '~mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_('mozilla.org' in report['url']) kwargs = { 'url': '^.com', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['url'], 'http://www.example.com') # Test user_comments kwargs = { 'user_comments': '~love', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['user_comments'], 'I love WaterWolf') kwargs = { 'user_comments': '$WaterWolf', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_( res['hits'][0]['user_comments'], 'WaterWolf is so bad' ) kwargs = { 'user_comments': '__null__', } res = self.api.get(**kwargs) eq_(res['total'], 19) eq_(res['hits'][0]['signature'], 'js::break_your_browser') for hit in res['hits']: eq_(hit['user_comments'], '') # Test address kwargs = { 'address': '^0', } res = self.api.get(**kwargs) eq_(res['total'], 21) kwargs = { 'address': '~a2', } res = self.api.get(**kwargs) eq_(res['total'], 1) # Test android_model kwargs = { 'android_model': '~PediaMad', } res = self.api.get(**kwargs) eq_(res['total'], 1) kwargs = { 'android_model': '=PediaMad 17 Heavy', } res = self.api.get(**kwargs) eq_(res['total'], 1) def test_get_with_facets(self): """Test a search with facets returns expected results. """ # Test several facets kwargs = { '_facets': ['signature', 'platform'] } res = self.api.get(**kwargs) ok_('facets' in res) ok_('signature' in res['facets']) expected_signatures = [ {'term': 'js::break_your_browser', 'count': 20}, {'term': 'my_bad', 'count': 1}, ] eq_(res['facets']['signature'], expected_signatures) ok_('platform' in res['facets']) expected_platforms = [ {'term': 'Linux', 'count': 20}, {'term': 'Windows NT', 'count': 1}, ] eq_(res['facets']['platform'], expected_platforms) # Test one facet with filters kwargs = { '_facets': ['release_channel'], 'release_channel': 'aurora', } res = self.api.get(**kwargs) ok_('release_channel' in res['facets']) expected_signatures = [ {'term': 'aurora', 'count': 1}, ] eq_(res['facets']['release_channel'], expected_signatures) # Test one facet with a different filter kwargs = { '_facets': ['release_channel'], 'platform': 'linux', } res = self.api.get(**kwargs) ok_('release_channel' in res['facets']) expected_signatures = [ {'term': 'release', 'count': 19}, {'term': 'aurora', 'count': 1}, ] eq_(res['facets']['release_channel'], expected_signatures) # Test errors assert_raises( BadArgumentError, self.api.get, _facets=['unkownfield'] ) def test_get_with_pagination(self): """Test a search with pagination returns expected results. """ kwargs = { '_results_number': '10', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 10) kwargs = { '_results_number': '10', '_results_offset': '10', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 10) kwargs = { '_results_number': '10', '_results_offset': '15', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 6) kwargs = { '_results_number': '10', '_results_offset': '30', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 0) def test_get_with_not_operator(self): """Test a search with a few NOT operators. """ # Test signature kwargs = { 'signature': ['js', 'break_your_browser'], } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test contains mode kwargs = { 'signature': '!~bad', } res = self.api.get(**kwargs) eq_(res['total'], 20) # - Test is_exactly mode kwargs = { 'signature': '!=js::break_your_browser', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode kwargs = { 'signature': '!$js', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test ends_with mode kwargs = { 'signature': '!^browser', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # Test build id kwargs = { 'build_id': '!<1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] > 1234567889) kwargs = { 'build_id': '!>1234567889', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_(res['hits'][0]['build_id'] < 1234567890) kwargs = { 'build_id': '!<=1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 0) @mock.patch( 'socorro.external.elasticsearch.supersearch.SuperSearch.get_indexes' ) def test_list_of_indices(self, mocked_get_indexes): """Test that unexisting indices are handled correctly. """ mocked_get_indexes.return_value = ['socorro_unknown'] res = self.api.get() res_expected = { 'hits': [], 'total': 0, 'facets': {}, } eq_(res, res_expected) mocked_get_indexes.return_value = [ 'socorro_integration_test', 'something_that_does_not_exist', 'another_one' ] res = self.api.get() ok_('total' in res) eq_(res['total'], 21) ok_('hits' in res) eq_(len(res['hits']), res['total']) ok_('facets' in res) ok_('signature' in res['facets']) def test_return_query_mode(self): kwargs = { 'signature': ['js', 'break_your_browser'], '_return_query': 'true' } res = self.api.get(**kwargs) ok_('query' in res) ok_('indices' in res) query = res['query'] ok_('filter' in query) ok_('facets' in query) ok_('size' in query)
def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = { 'signature': 'my_bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # Test product args = { 'product': 'EarthRaccoon', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['product'], 'EarthRaccoon') # Test version args = { 'version': '2.0', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['version'], '2.0') # Test release_channel args = { 'release_channel': 'aurora', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['release_channel'], 'aurora') # Test platform args = { 'platform': 'Windows', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['platform'], 'Windows NT') # Test build_id args = { 'build_id': '987654321', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['build_id'], 987654321) # Test reason args = { 'reason': 'MOZALLOC_WENT_WRONG', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertEqual(res['hits'][0]['reason'], 'MOZALLOC_WENT_WRONG') args = { 'reason': ['very_bad_exception'], } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['reason'], 'VERY_BAD_EXCEPTION') # Test process_type args = { 'process_type': 'plugin', } res = api.get(**args) self.assertEqual(res['total'], 3) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['process_type'], 'plugin') # Test url args = { 'url': 'https://mozilla.org', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('mozilla.org' in res['hits'][0]['url']) # Test user_comments args = { 'user_comments': 'WaterWolf', } res = api.get(**args) self.assertEqual(res['total'], 2) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('WaterWolf' in res['hits'][0]['user_comments']) # Test address args = { 'address': '0x0', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue('0x0' in res['hits'][0]['address']) # Test accessibility args = { 'accessibility': False, } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertTrue(not res['hits'][0]['accessibility']) args = { 'accessibility': 'True', } res = api.get(**args) self.assertEqual(res['total'], 7) self.assertTrue(res['hits'][0]['accessibility']) # Test b2g_os_version args = { 'b2g_os_version': '1.3', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['b2g_os_version'], '1.3') # Test bios_manufacturer args = { 'bios_manufacturer': 'aidivn', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['bios_manufacturer'], 'aidivn') # Test is_garbage_collecting args = { 'is_garbage_collecting': True, } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertTrue(res['hits'][0]['is_garbage_collecting']) # Test vendor args = { 'vendor': 'gnusmas', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['vendor'], 'gnusmas') # Test useragent_locale args = { 'useragent_locale': 'fr', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['useragent_locale'], 'fr')
def test_get_with_not_operator(self): """Test a search with a few NOT operators. """ with _get_config_manager().context() as config: api = SuperSearch(config) # Test signature args = { 'signature': ['js', 'break_your_browser'], } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test contains mode args = { 'signature': '!~bad', } res = api.get(**args) self.assertEqual(res['total'], 20) # - Test is_exactly mode args = { 'signature': '!=js::break_your_browser', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode args = { 'signature': '!$js', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test ends_with mode args = { 'signature': '!^browser', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # Test build id args = { 'build_id': '!<1234567890', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build'] > 1234567889) args = { 'build_id': '!>1234567889', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue(res['hits'][0]['build'] < 1234567890) args = { 'build_id': '!<=1234567890', } res = api.get(**args) self.assertEqual(res['total'], 0)
def test_get_with_facets(self): """Test a search with facets returns expected results. """ with _get_config_manager().context() as config: api = SuperSearch(config) # Test several facets args = { '_facets': ['signature', 'platform'] } res = api.get(**args) self.assertTrue('facets' in res) self.assertTrue('signature' in res['facets']) expected_signatures = [ {'term': 'js::break_your_browser', 'count': 20}, {'term': 'my_bad', 'count': 1}, ] self.assertEqual(res['facets']['signature'], expected_signatures) self.assertTrue('platform' in res['facets']) expected_platforms = [ {'term': 'linux', 'count': 20}, {'term': 'windows', 'count': 1}, {'term': 'nt', 'count': 1}, ] self.assertEqual(res['facets']['platform'], expected_platforms) # Test one facet with filters args = { '_facets': ['release_channel'], 'release_channel': 'aurora', } res = api.get(**args) self.assertTrue('release_channel' in res['facets']) expected_signatures = [ {'term': 'aurora', 'count': 1}, ] self.assertEqual(res['facets']['release_channel'], expected_signatures) # Test one facet with a different filter args = { '_facets': ['release_channel'], 'platform': 'linux', } res = api.get(**args) self.assertTrue('release_channel' in res['facets']) expected_signatures = [ {'term': 'release', 'count': 19}, {'term': 'aurora', 'count': 1}, ] self.assertEqual(res['facets']['release_channel'], expected_signatures) # Test errors self.assertRaises( BadArgumentError, api.get, _facets=['unkownfield'] )
def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = {"signature": "my_bad"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "my_bad") # Test product args = {"product": "EarthRaccoon"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["product"], "EarthRaccoon") # Test version args = {"version": "2.0"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["version"], "2.0") # Test release_channel args = {"release_channel": "aurora"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["release_channel"], "aurora") # Test platform args = {"platform": "Windows"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["platform"], "Windows NT") # Test build_id args = {"build_id": "987654321"} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["build_id"], 987654321) # Test reason args = {"reason": "MOZALLOC_WENT_WRONG"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertEqual(res["hits"][0]["reason"], "MOZALLOC_WENT_WRONG") args = {"reason": ["very_bad_exception"]} res = api.get(**args) self.assertEqual(res["total"], 1) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["reason"], "VERY_BAD_EXCEPTION") # Test process_type args = {"process_type": "plugin"} res = api.get(**args) self.assertEqual(res["total"], 3) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertEqual(res["hits"][0]["process_type"], "plugin") # Test url args = {"url": "https://mozilla.org"} res = api.get(**args) self.assertEqual(res["total"], 19) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertTrue("mozilla.org" in res["hits"][0]["url"]) # Test user_comments args = {"user_comments": "WaterWolf"} res = api.get(**args) self.assertEqual(res["total"], 2) self.assertEqual(res["hits"][0]["signature"], "js::break_your_browser") self.assertTrue("WaterWolf" in res["hits"][0]["user_comments"]) # Test address args = {"address": "0x0"} res = api.get(**args) self.assertEqual(res["total"], 20) self.assertTrue("0x0" in res["hits"][0]["address"])
class IntegrationTestSuperSearch(ElasticSearchTestCase): """Test SuperSearch with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearch, self).setUp() config = self.get_config_context() self.api = SuperSearch(config=config) self.storage = crashstorage.ElasticSearchCrashStorage(config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() now = datetimeutil.utc_now() yesterday = now - datetime.timedelta(days=1) yesterday = datetimeutil.date_to_string(yesterday) last_month = now - datetime.timedelta(weeks=4) last_month = datetimeutil.date_to_string(last_month) # insert data into elasticsearch default_crash_report = { 'uuid': 100, 'address': '0x0', 'signature': 'js::break_your_browser', 'date_processed': yesterday, 'product': 'WaterWolf', 'version': '1.0', 'release_channel': 'release', 'os_name': 'Linux', 'build': 1234567890, 'reason': 'MOZALLOC_WENT_WRONG', 'hangid': None, 'process_type': None, 'email': '*****@*****.**', 'url': 'https://mozilla.org', 'user_comments': '', 'install_age': 0, } default_raw_crash_report = { 'Accessibility': True, 'AvailableVirtualMemory': 10211743, 'B2G_OS_Version': '1.1.203448', 'BIOS_Manufacturer': 'SUSA', 'IsGarbageCollecting': False, 'Vendor': 'mozilla', 'useragent_locale': 'en-US', } self.storage.save_raw_and_processed(default_raw_crash_report, None, default_crash_report, default_crash_report['uuid']) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Accessibility=False), None, dict(default_crash_report, uuid=1, product='EarthRaccoon'), 1) self.storage.save_raw_and_processed( dict(default_raw_crash_report, AvailableVirtualMemory=0), None, dict(default_crash_report, uuid=2, version='2.0'), 2) self.storage.save_raw_and_processed( dict(default_raw_crash_report, B2G_OS_Version='1.3'), None, dict(default_crash_report, uuid=3, release_channel='aurora'), 3) self.storage.save_raw_and_processed( dict(default_raw_crash_report, BIOS_Manufacturer='aidivn'), None, dict(default_crash_report, uuid=4, os_name='Windows NT'), 4) self.storage.save_raw_and_processed( dict(default_raw_crash_report, IsGarbageCollecting=True), None, dict(default_crash_report, uuid=5, build=987654321), 5) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Vendor='gnusmas'), None, dict(default_crash_report, uuid=6, reason='VERY_BAD_EXCEPTION'), 6) self.storage.save_raw_and_processed( dict(default_raw_crash_report, useragent_locale='fr'), None, dict(default_crash_report, uuid=7, hangid=12), 7) self.storage.save_raw_and_processed( dict(default_raw_crash_report, Android_Model='PediaMad 17 Heavy'), None, dict(default_crash_report, uuid=8, process_type='plugin'), 8) self.storage.save_processed( dict(default_crash_report, uuid=9, signature='my_bad')) self.storage.save_processed( dict( default_crash_report, uuid=10, date_processed=last_month, signature='my_little_signature', )) # for plugin terms test self.storage.save_processed( dict( default_crash_report, uuid=11, product='PluginSoft', process_type='plugin', PluginFilename='carly.dll', PluginName='Hey I just met you', PluginVersion='1.2', )) self.storage.save_processed( dict( default_crash_report, uuid=12, product='PluginSoft', process_type='plugin', PluginFilename='hey.dll', PluginName='Hey Plugin', PluginVersion='10.7.0.2a', )) self.storage.save_processed( dict(default_crash_report, uuid=13, email='*****@*****.**')) self.storage.save_processed( dict(default_crash_report, uuid=14, email='*****@*****.**')) self.storage.save_processed( dict(default_crash_report, uuid=15, email='*****@*****.**')) self.storage.save_processed( dict(default_crash_report, uuid=16, install_age=87234)) self.storage.save_processed( dict(default_crash_report, uuid=17, url='http://www.mozilla.org')) self.storage.save_processed( dict(default_crash_report, uuid=18, url='http://www.example.com')) self.storage.save_processed( dict( default_crash_report, uuid=19, user_comments='I love WaterWolf', )) self.storage.save_processed( dict( default_crash_report, uuid=20, user_comments='WaterWolf is so bad', )) self.storage.save_processed( dict(default_crash_report, uuid=21, address='0xa2e4509ca0')) # As indexing is asynchronous, we need to force elasticsearch to # make the newly created content searchable before we run the tests self.storage.es.refresh() def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) super(IntegrationTestSuperSearch, self).tearDown() def test_get(self): """Test a search with default values returns the right structure. """ res = self.api.get() ok_('total' in res) eq_(res['total'], 21) ok_('hits' in res) eq_(len(res['hits']), res['total']) ok_('facets' in res) ok_('signature' in res['facets']) expected_signatures = [ { 'term': 'js::break_your_browser', 'count': 20 }, { 'term': 'my_bad', 'count': 1 }, ] eq_(res['facets']['signature'], expected_signatures) # Test fields are being renamed ok_('date' in res['hits'][0]) # date_processed > date ok_('build_id' in res['hits'][0]) # build > build_id ok_('platform' in res['hits'][0]) # os_name > platform def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ # Test signature kwargs = { 'signature': 'my_bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # Test product kwargs = { 'product': 'EarthRaccoon', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['product'], 'EarthRaccoon') # Test version kwargs = { 'version': '2.0', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['version'], '2.0') # Test release_channel kwargs = { 'release_channel': 'aurora', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['release_channel'], 'aurora') # Test platform kwargs = { 'platform': 'Windows', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['platform'], 'Windows NT') # Test build_id kwargs = { 'build_id': '987654321', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['build_id'], 987654321) # Test reason kwargs = { 'reason': 'MOZALLOC_WENT_WRONG', } res = self.api.get(**kwargs) eq_(res['total'], 20) eq_(res['hits'][0]['reason'], 'MOZALLOC_WENT_WRONG') kwargs = { 'reason': ['very_bad_exception'], } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['reason'], 'VERY_BAD_EXCEPTION') # Test process_type kwargs = { 'process_type': 'plugin', } res = self.api.get(**kwargs) eq_(res['total'], 3) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['process_type'], 'plugin') # Test url kwargs = { 'url': 'https://mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 19) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_('mozilla.org' in res['hits'][0]['url']) # Test user_comments kwargs = { 'user_comments': 'WaterWolf', } res = self.api.get(**kwargs) eq_(res['total'], 2) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_('WaterWolf' in res['hits'][0]['user_comments']) # Test address kwargs = { 'address': '0x0', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_('0x0' in res['hits'][0]['address']) # Test accessibility kwargs = { 'accessibility': False, } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(not res['hits'][0]['accessibility']) kwargs = { 'accessibility': 'True', } res = self.api.get(**kwargs) eq_(res['total'], 8) ok_(res['hits'][0]['accessibility']) # Test b2g_os_version kwargs = { 'b2g_os_version': '1.3', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['b2g_os_version'], '1.3') # Test bios_manufacturer kwargs = { 'bios_manufacturer': 'aidivn', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['bios_manufacturer'], 'aidivn') # Test is_garbage_collecting kwargs = { 'is_garbage_collecting': True, } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(res['hits'][0]['is_garbage_collecting']) # Test vendor kwargs = { 'vendor': 'gnusmas', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['vendor'], 'gnusmas') # Test useragent_locale kwargs = { 'useragent_locale': 'fr', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['useragent_locale'], 'fr') def test_get_with_range_operators(self): """Test a search with several filters and operators returns expected results. """ # Test date now = datetimeutil.utc_now() lastweek = now - datetime.timedelta(days=7) lastmonth = lastweek - datetime.timedelta(weeks=4) kwargs = { 'date': [ '<%s' % lastweek, '>=%s' % lastmonth, ] } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_little_signature') # Test build id kwargs = { 'build_id': '<1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_(res['hits'][0]['build_id'] < 1234567890) kwargs = { 'build_id': '>1234567889', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] > 1234567889) kwargs = { 'build_id': '<=1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 21) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] <= 1234567890) # Test available_virtual_memory kwargs = { 'available_virtual_memory': '>=1', } res = self.api.get(**kwargs) eq_(res['total'], 8) for report in res['hits']: ok_(report['available_virtual_memory'] >= 1) kwargs = { 'available_virtual_memory': '<1', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['available_virtual_memory'], 0) def test_get_with_string_operators(self): """Test a search with several filters and operators returns expected results. """ # Test signature kwargs = { 'signature': ['js', 'break_your_browser'], } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test contains mode kwargs = { 'signature': '~bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') kwargs = { 'signature': '~js::break', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test is_exactly mode kwargs = { 'signature': '=js::break_your_browser', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') kwargs = { 'signature': '=my_bad', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode kwargs = { 'signature': '$js', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test ends_with mode kwargs = { 'signature': '^browser', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # Test email kwargs = { 'email': '*****@*****.**', } res = self.api.get(**kwargs) eq_(res['total'], 1) ok_(res['hits']) eq_(res['hits'][0]['email'], '*****@*****.**') kwargs = { 'email': '~mail.com', } res = self.api.get(**kwargs) eq_(res['total'], 19) ok_(res['hits']) for report in res['hits']: ok_('@' in report['email']) ok_('mail.com' in report['email']) kwargs = { 'email': '$sauron@', } res = self.api.get(**kwargs) eq_(res['total'], 2) ok_(res['hits']) for report in res['hits']: ok_('sauron@' in report['email']) # Test url kwargs = { 'url': 'https://mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 19) kwargs = { 'url': '~mozilla.org', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_('mozilla.org' in report['url']) kwargs = { 'url': '^.com', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['url'], 'http://www.example.com') # Test user_comments kwargs = { 'user_comments': '~love', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['user_comments'], 'I love WaterWolf') kwargs = { 'user_comments': '$WaterWolf', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') eq_(res['hits'][0]['user_comments'], 'WaterWolf is so bad') kwargs = { 'user_comments': '__null__', } res = self.api.get(**kwargs) eq_(res['total'], 19) eq_(res['hits'][0]['signature'], 'js::break_your_browser') for hit in res['hits']: eq_(hit['user_comments'], '') # Test address kwargs = { 'address': '^0', } res = self.api.get(**kwargs) eq_(res['total'], 21) kwargs = { 'address': '~a2', } res = self.api.get(**kwargs) eq_(res['total'], 1) # Test android_model kwargs = { 'android_model': '~PediaMad', } res = self.api.get(**kwargs) eq_(res['total'], 1) kwargs = { 'android_model': '=PediaMad 17 Heavy', } res = self.api.get(**kwargs) eq_(res['total'], 1) def test_get_with_facets(self): """Test a search with facets returns expected results. """ # Test several facets kwargs = {'_facets': ['signature', 'platform']} res = self.api.get(**kwargs) ok_('facets' in res) ok_('signature' in res['facets']) expected_signatures = [ { 'term': 'js::break_your_browser', 'count': 20 }, { 'term': 'my_bad', 'count': 1 }, ] eq_(res['facets']['signature'], expected_signatures) ok_('platform' in res['facets']) expected_platforms = [ { 'term': 'Linux', 'count': 20 }, { 'term': 'Windows NT', 'count': 1 }, ] eq_(res['facets']['platform'], expected_platforms) # Test one facet with filters kwargs = { '_facets': ['release_channel'], 'release_channel': 'aurora', } res = self.api.get(**kwargs) ok_('release_channel' in res['facets']) expected_signatures = [ { 'term': 'aurora', 'count': 1 }, ] eq_(res['facets']['release_channel'], expected_signatures) # Test one facet with a different filter kwargs = { '_facets': ['release_channel'], 'platform': 'linux', } res = self.api.get(**kwargs) ok_('release_channel' in res['facets']) expected_signatures = [ { 'term': 'release', 'count': 19 }, { 'term': 'aurora', 'count': 1 }, ] eq_(res['facets']['release_channel'], expected_signatures) # Test errors assert_raises(BadArgumentError, self.api.get, _facets=['unkownfield']) def test_get_with_pagination(self): """Test a search with pagination returns expected results. """ kwargs = { '_results_number': '10', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 10) kwargs = { '_results_number': '10', '_results_offset': '10', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 10) kwargs = { '_results_number': '10', '_results_offset': '15', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 6) kwargs = { '_results_number': '10', '_results_offset': '30', } res = self.api.get(**kwargs) eq_(res['total'], 21) eq_(len(res['hits']), 0) def test_get_with_not_operator(self): """Test a search with a few NOT operators. """ # Test signature kwargs = { 'signature': ['js', 'break_your_browser'], } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: eq_(report['signature'], 'js::break_your_browser') # - Test contains mode kwargs = { 'signature': '!~bad', } res = self.api.get(**kwargs) eq_(res['total'], 20) # - Test is_exactly mode kwargs = { 'signature': '!=js::break_your_browser', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode kwargs = { 'signature': '!$js', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # - Test ends_with mode kwargs = { 'signature': '!^browser', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'my_bad') # Test build id kwargs = { 'build_id': '!<1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 20) ok_(res['hits']) for report in res['hits']: ok_(report['build_id'] > 1234567889) kwargs = { 'build_id': '!>1234567889', } res = self.api.get(**kwargs) eq_(res['total'], 1) eq_(res['hits'][0]['signature'], 'js::break_your_browser') ok_(res['hits'][0]['build_id'] < 1234567890) kwargs = { 'build_id': '!<=1234567890', } res = self.api.get(**kwargs) eq_(res['total'], 0) @mock.patch( 'socorro.external.elasticsearch.supersearch.SuperSearch.get_indexes') def test_list_of_indices(self, mocked_get_indexes): """Test that unexisting indices are handled correctly. """ mocked_get_indexes.return_value = ['socorro_unknown'] res = self.api.get() res_expected = { 'hits': [], 'total': 0, 'facets': {}, } eq_(res, res_expected) mocked_get_indexes.return_value = [ 'socorro_integration_test', 'something_that_does_not_exist', 'another_one' ] res = self.api.get() ok_('total' in res) eq_(res['total'], 21) ok_('hits' in res) eq_(len(res['hits']), res['total']) ok_('facets' in res) ok_('signature' in res['facets']) def test_return_query_mode(self): kwargs = { 'signature': ['js', 'break_your_browser'], '_return_query': 'true' } res = self.api.get(**kwargs) ok_('query' in res) ok_('indices' in res) query = res['query'] ok_('filter' in query) ok_('facets' in query) ok_('size' in query)
def test_get_individual_filters(self): """Test a search with single filters returns expected results. """ with _get_config_manager().context() as config: api = SuperSearch(config) # Test signature args = { 'signature': 'my_bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # Test product args = { 'product': 'EarthRaccoon', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['product'], 'EarthRaccoon') # Test version args = { 'version': '2.0', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['version'], '2.0') # Test release_channel args = { 'release_channel': 'aurora', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['release_channel'], 'aurora') # Test platform args = { 'platform': 'Windows', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['os_name'], 'Windows NT') # Test build_id args = { 'build_id': '987654321', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['build'], 987654321) # Test reason args = { 'reason': 'MOZALLOC_WENT_WRONG', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertEqual(res['hits'][0]['reason'], 'MOZALLOC_WENT_WRONG') args = { 'reason': ['very_bad_exception'], } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['reason'], 'VERY_BAD_EXCEPTION') # Test process_type args = { 'process_type': 'plugin', } res = api.get(**args) self.assertEqual(res['total'], 3) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['process_type'], 'plugin') # Test url args = { 'url': 'mozilla', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('mozilla.org' in res['hits'][0]['url']) # Test user_comments args = { 'user_comments': 'WaterWolf', } res = api.get(**args) self.assertEqual(res['total'], 2) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue('WaterWolf' in res['hits'][0]['user_comments']) # Test address args = { 'address': '0x0', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue('0x0' in res['hits'][0]['address'])
class IntegrationTestSettings(ElasticSearchTestCase): """Test the settings and mappings used in elasticsearch, through the supersearch service. """ def setUp(self): super(IntegrationTestSettings, self).setUp() config = self.get_config_context() self.storage = crashstorage.ElasticSearchCrashStorage(config) self.api = SuperSearch(config=config) # clear the indices cache so the index is created on every test self.storage.indices_cache = set() self.now = utc_now() # Create the index that will be used. es_index = self.storage.get_index_for_crash(self.now) self.storage.create_socorro_index(es_index) # This an ugly hack to give elasticsearch some time to finish creating # the new index. It is needed for jenkins only, because we have a # special case here where we index only one or two documents before # querying. Other tests are not affected. # TODO: try to remove it, or at least understand why it is needed. time.sleep(1) def tearDown(self): # clear the test index config = self.get_config_context() self.storage.es.delete_index(config.webapi.elasticsearch_index) super(IntegrationTestSettings, self).tearDown() def test_dump_field(self): """Verify that the 'dump' field can be queried as expected. """ # Index some data. processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'dump': EXAMPLE_DUMP, } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(dump='~family') self.assertEqual(res['total'], 1) # Several words, with upper case. res = self.api.get(dump='~Windows NT') self.assertEqual(res['total'], 1) def test_cpu_info_field(self): """Verify that the 'cpu_info' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'cpu_info': 'GenuineIntel family 6 model 15 stepping 13', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Simple test with one word, no upper case. res = self.api.get(cpu_info='~model') self.assertEqual(res['total'], 1) self.assertTrue('model' in res['hits'][0]['cpu_info']) # Several words, with upper case, 'starts with' mode. res = self.api.get(cpu_info='$GenuineIntel family') self.assertEqual(res['total'], 1) self.assertTrue('GenuineIntel family' in res['hits'][0]['cpu_info']) def test_dom_ipc_enabled_field(self): """Verify that the 'dom_ipc_enabled' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': True, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': False, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, } raw_crash = { 'DOMIPCEnabled': None, } self.storage.save_raw_and_processed( raw_crash, None, processed_crash, processed_crash['uuid'] ) self.storage.es.refresh() res = self.api.get(dom_ipc_enabled='true') self.assertEqual(res['total'], 1) self.assertTrue(res['hits'][0]['dom_ipc_enabled']) res = self.api.get(dom_ipc_enabled='false') self.assertEqual(res['total'], 2) def test_platform_field(self): """Verify that the 'platform' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120102', 'date_processed': self.now, 'os_name': 'Mac OS X', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(platform='Mac OS X') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['platform'], 'Mac OS X') def test_app_notes_field(self): """Verify that the 'app_notes' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120103', 'date_processed': self.now, 'app_notes': 'there is a cycle collector fault here', } self.storage.save_processed(processed_crash) self.storage.es.refresh() # Testing the phrase mode, when a term query contains white spaces. res = self.api.get(app_notes='cycle collector fault') self.assertEqual(res['total'], 1) self.assertTrue('cycle collector fault' in res['hits'][0]['app_notes']) def test_process_type_field(self): """Verify that the 'process_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'process_type': 'plugin', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'process_type': None, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(process_type='plugin') self.assertEqual(res['total'], 1) self.assertTrue('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type='browser') self.assertEqual(res['total'], 1) # In the case of a 'browser' crash, the process_type is None and thus # is not returned. self.assertTrue('process_type' not in res['hits'][0]) res = self.api.get(process_type='!browser') self.assertEqual(res['total'], 1) self.assertTrue('plugin' in res['hits'][0]['process_type']) res = self.api.get(process_type=['plugin', 'browser']) self.assertEqual(res['total'], 2) def test_hang_type_field(self): """Verify that the 'hang_type' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'hang_type': 0, } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'hang_type': 1, } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(hang_type='hang') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type='crash') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['hang_type'], 0) res = self.api.get(hang_type='!crash') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['hang_type'], 1) res = self.api.get(hang_type=['crash', 'hang']) self.assertEqual(res['total'], 2) def test_exploitability_field(self): """Verify that the 'exploitability' field can be queried as expected. """ processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120100', 'date_processed': self.now, 'exploitability': 'high', } self.storage.save_processed(processed_crash) processed_crash = { 'uuid': '06a0c9b5-0381-42ce-855a-ccaaa2120101', 'date_processed': self.now, 'exploitability': 'unknown', } self.storage.save_processed(processed_crash) self.storage.es.refresh() res = self.api.get(exploitability='high') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['exploitability'], 'high') res = self.api.get(exploitability='unknown') self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['exploitability'], 'unknown') res = self.api.get(exploitability=['high', 'unknown']) self.assertEqual(res['total'], 2)
def test_get_with_not_operator(self): """Test a search with a few NOT operators. """ with _get_config_manager(self.config).context() as config: api = SuperSearch(config) # Test signature args = { 'signature': ['js', 'break_your_browser'], } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test contains mode args = { 'signature': '!~bad', } res = api.get(**args) self.assertEqual(res['total'], 20) # - Test is_exactly mode args = { 'signature': '!=js::break_your_browser', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode args = { 'signature': '!$js', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test ends_with mode args = { 'signature': '!^browser', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # Test build id args = { 'build_id': '!<1234567890', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue(report['build_id'] > 1234567889) args = { 'build_id': '!>1234567889', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertTrue(res['hits'][0]['build_id'] < 1234567890) args = { 'build_id': '!<=1234567890', } res = api.get(**args) self.assertEqual(res['total'], 0)
def test_get_with_string_operators(self): """Test a search with several filters and operators returns expected results. """ with _get_config_manager().context() as config: api = SuperSearch(config) # Test signature args = { 'signature': ['js', 'break_your_browser'], } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test contains mode args = { 'signature': '~bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') args = { 'signature': '~js::break', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test is_exactly mode args = { 'signature': '=js::break_your_browser', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') args = { 'signature': '=my_bad', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'my_bad') # - Test starts_with mode args = { 'signature': '$js', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # - Test ends_with mode args = { 'signature': '^browser', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertEqual(report['signature'], 'js::break_your_browser') # Test email args = { 'email': ['gmail', 'hotmail'], } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('@' in report['email']) self.assertTrue('mail.com' in report['email']) args = { 'email': '~mail.com', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('@' in report['email']) self.assertTrue('mail.com' in report['email']) args = { 'email': '$sauron@', } res = api.get(**args) self.assertEqual(res['total'], 2) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('sauron@' in report['email']) # Test url args = { 'url': ['mozilla', 'www'], } res = api.get(**args) self.assertEqual(res['total'], 21) args = { 'url': '~mozilla.org', } res = api.get(**args) self.assertEqual(res['total'], 20) self.assertTrue(res['hits']) for report in res['hits']: self.assertTrue('mozilla.org' in report['url']) args = { 'url': '^.com', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['url'], 'http://www.example.com') # Test user_comments args = { 'user_comments': '~love', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual(res['hits'][0]['user_comments'], 'I love WaterWolf') args = { 'user_comments': '$WaterWolf', } res = api.get(**args) self.assertEqual(res['total'], 1) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') self.assertEqual( res['hits'][0]['user_comments'], 'WaterWolf is so bad' ) args = { 'user_comments': '__null__', } res = api.get(**args) self.assertEqual(res['total'], 19) self.assertEqual(res['hits'][0]['signature'], 'js::break_your_browser') for hit in res['hits']: self.assertEqual(hit['user_comments'], '') # Test address args = { 'address': '^0', } res = api.get(**args) self.assertEqual(res['total'], 21) args = { 'address': '~a2', } res = api.get(**args) self.assertEqual(res['total'], 1)