class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() # Create the supersearch fields. self.index_super_search_fields() self.api = SuperSearchFields(config=self.config) def tearDown(self): # Clear the test indices. self.index_client.delete( self.config.elasticsearch.elasticsearch_default_index ) super(IntegrationTestSuperSearchFields, self).tearDown() def test_get_fields(self): results = self.api.get_fields() eq_(results, SUPERSEARCH_FIELDS) def test_get_mapping(self): mapping = self.api.get_mapping()['mappings'] doctype = self.config.elasticsearch.elasticsearch_doctype ok_(doctype in mapping) properties = mapping[doctype]['properties'] ok_('processed_crash' in properties) ok_('raw_crash' in properties) # Check in_database_name is used. ok_('os_name' in properties['processed_crash']['properties']) ok_('platform' not in properties['processed_crash']['properties']) # Those fields have no `storage_mapping`. ok_('fake_field' not in properties['raw_crash']['properties']) # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={ 'name': 'fake_field', 'storage_mapping': { 'type': 'long' } })['mappings'] properties = mapping[doctype]['properties'] ok_('fake_field' in properties['raw_crash']['properties']) eq_( properties['raw_crash']['properties']['fake_field']['type'], 'long' )
class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() self.api = SuperSearchFields(config=self.config) self.api.get_fields = lambda: copy.deepcopy(FIELDS) def test_get_fields(self): results = self.api.get_fields() assert results == FIELDS def test_get_missing_fields(self): config = self.get_base_config( es_index='socorro_integration_test_%W' ) fake_mappings = [ { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { # Add a bunch of unknown fields. 'field_z': { 'type': 'string' }, 'namespace1': { 'type': 'object', 'properties': { 'field_a': { 'type': 'string' }, 'field_b': { 'type': 'long' } } }, 'namespace2': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_b': { 'type': 'long' } } } } }, # Add a few known fields that should not appear. 'processed_crash': { 'type': 'object', 'properties': { 'signature': { 'type': 'string' }, 'product': { 'type': 'string' }, } } } } } }, { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { 'namespace1': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_d': { 'type': 'long' } } } } } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ 'field_z', 'namespace1.field_a', 'namespace1.field_b', 'namespace1.subspace1.field_d', 'namespace2.subspace1.field_b', ] assert missing_fields['hits'] == expected assert missing_fields['total'] == 5 finally: for index in indices: self.index_client.delete(index=index) def test_get_mapping(self): mapping = self.api.get_mapping() doctype = self.config.elasticsearch.elasticsearch_doctype assert doctype in mapping properties = mapping[doctype]['properties'] assert 'processed_crash' in properties assert 'raw_crash' in properties processed_crash = properties['processed_crash']['properties'] # Check in_database_name is used. assert 'os_name' in processed_crash assert 'platform' not in processed_crash # Those fields have no `storage_mapping`. assert 'fake_field' not in properties['raw_crash']['properties'] # Those fields have a `storage_mapping`. assert processed_crash['release_channel'] == {'analyzer': 'keyword', 'type': 'string'} # Test nested objects. assert 'json_dump' in processed_crash assert 'properties' in processed_crash['json_dump'] assert 'write_combine_size' in processed_crash['json_dump']['properties'] assert processed_crash['json_dump']['properties']['write_combine_size'] == {'type': 'long'} # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={ 'name': 'fake_field', 'namespace': 'raw_crash', 'in_database_name': 'fake_field', 'storage_mapping': { 'type': 'long' } }) properties = mapping[doctype]['properties'] assert 'fake_field' in properties['raw_crash']['properties'] assert properties['raw_crash']['properties']['fake_field']['type'] == 'long' def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() assert self.api.test_mapping(mapping) is None # Insert an invalid storage mapping. mapping = self.api.get_mapping({ 'name': 'fake_field', 'namespace': 'raw_crash', 'in_database_name': 'fake_field', 'storage_mapping': { 'type': 'unkwown' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({ 'date_processed': datetimeutil.utc_now(), 'product': 'WaterWolf', }) self.refresh_index() mapping = self.api.get_mapping({ 'name': 'product', 'storage_mapping': { 'type': 'long' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping)
class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() self.api = SuperSearchFields(config=self.config) def test_get_fields(self): results = self.api.get_fields() eq_(results, SUPERSEARCH_FIELDS) def test_create_field(self): # Test with all parameters set. params = { 'name': 'plotfarm', 'data_validation_type': 'str', 'default_value': None, 'description': 'a plotfarm like Lunix or Wondiws', 'form_field_choices': ['lun', 'won', 'cam'], 'has_full_version': True, 'in_database_name': 'os_name', 'is_exposed': True, 'is_returned': True, 'is_mandatory': False, 'query_type': 'str', 'namespace': 'processed_crash', 'permissions_needed': ['view_plotfarm'], 'storage_mapping': {"type": "multi_field"}, } res = self.api.create_field(**params) ok_(res) field = self.connection.get( index=self.config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', id='plotfarm', ) field = field['_source'] eq_(sorted(field.keys()), sorted(params.keys())) for key in field.keys(): eq_(field[key], params[key]) # Test default values. res = self.api.create_field( name='brand_new_field', in_database_name='brand_new_field', namespace='processed_crash', ) ok_(res) ok_( self.connection.get( index=self.config.webapi.elasticsearch_default_index, doc_type='supersearch_fields', id='brand_new_field', ) ) # Test errors. # `name` is missing. assert_raises( MissingArgumentError, self.api.create_field, in_database_name='something', ) # `in_database_name` is missing. assert_raises( MissingArgumentError, self.api.create_field, name='something', ) # Field already exists. assert_raises( BadArgumentError, self.api.create_field, name='product', in_database_name='product', namespace='processed_crash', ) # Test logging. res = self.api.create_field( name='what_a_field', in_database_name='what_a_field', namespace='processed_crash', storage_mapping='{"type": "long"}', ) ok_(res) self.api.config.logger.info.assert_called_with( 'elasticsearch mapping changed for field "%s", ' 'added new mapping "%s"', 'what_a_field', {u'type': u'long'}, ) def test_update_field(self): # Let's create a field first. assert self.api.create_field( name='super_field', in_database_name='super_field', namespace='superspace', description='inaccurate description', permissions_needed=['view_nothing'], storage_mapping={'type': 'boolean', 'null_value': False} ) # Now let's update that field a little. res = self.api.update_field( name='super_field', description='very accurate description', storage_mapping={'type': 'long', 'analyzer': 'keyword'}, ) ok_(res) # Test logging. self.api.config.logger.info.assert_called_with( 'elasticsearch mapping changed for field "%s", ' 'was "%s", now "%s"', 'super_field', {'type': 'boolean', 'null_value': False}, {'type': 'long', 'analyzer': 'keyword'}, ) field = self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='super_field', ) field = field['_source'] # Verify the changes were taken into account. eq_(field['description'], 'very accurate description') eq_(field['storage_mapping'], {'type': 'long', 'analyzer': 'keyword'}) # Verify other values did not change. eq_(field['permissions_needed'], ['view_nothing']) eq_(field['in_database_name'], 'super_field') eq_(field['namespace'], 'superspace') # Test errors. assert_raises( MissingArgumentError, self.api.update_field, ) # `name` is missing assert_raises( ResourceNotFound, self.api.update_field, name='unkownfield', ) def test_delete_field(self): self.api.delete_field(name='product') ok_( self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='signature', ) ) assert_raises( elasticsearch.exceptions.NotFoundError, self.connection.get, index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='product', ) @minimum_es_version('1.0') def test_get_missing_fields(self): config = self.get_mware_config( es_index='socorro_integration_test_%W' ) fake_mappings = [ { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { # Add a bunch of unknown fields. 'field_z': { 'type': 'string' }, 'namespace1': { 'type': 'object', 'properties': { 'field_a': { 'type': 'string' }, 'field_b': { 'type': 'long' } } }, 'namespace2': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_b': { 'type': 'long' } } } } }, # Add a few known fields that should not appear. 'processed_crash': { 'type': 'object', 'properties': { 'signature': { 'type': 'string' }, 'product': { 'type': 'string' }, } } } } } }, { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { 'namespace1': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_d': { 'type': 'long' } } } } } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ 'field_z', 'namespace1.field_a', 'namespace1.field_b', 'namespace1.subspace1.field_d', 'namespace2.subspace1.field_b', ] eq_(missing_fields['hits'], expected) eq_(missing_fields['total'], 5) finally: for index in indices: self.index_client.delete(index=index) def test_get_mapping(self): mapping = self.api.get_mapping()['mappings'] doctype = self.config.elasticsearch.elasticsearch_doctype ok_(doctype in mapping) properties = mapping[doctype]['properties'] ok_('processed_crash' in properties) ok_('raw_crash' in properties) processed_crash = properties['processed_crash']['properties'] # Check in_database_name is used. ok_('os_name' in processed_crash) ok_('platform' not in processed_crash) # Those fields have no `storage_mapping`. ok_('fake_field' not in properties['raw_crash']['properties']) # Those fields have a `storage_mapping`. eq_(processed_crash['release_channel'], {'type': 'string'}) # Test nested objects. ok_('json_dump' in processed_crash) ok_('properties' in processed_crash['json_dump']) ok_('write_combine_size' in processed_crash['json_dump']['properties']) eq_( processed_crash['json_dump']['properties']['write_combine_size'], {'type': 'long'} ) # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={ 'name': 'fake_field', 'storage_mapping': { 'type': 'long' } })['mappings'] properties = mapping[doctype]['properties'] ok_('fake_field' in properties['raw_crash']['properties']) eq_( properties['raw_crash']['properties']['fake_field']['type'], 'long' ) def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() ok_(self.api.test_mapping(mapping) is None) # Insert an invalid storage mapping. mapping = self.api.get_mapping({ 'name': 'fake_field', 'storage_mapping': { 'type': 'unkwown' } }) assert_raises( BadArgumentError, self.api.test_mapping, mapping, ) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({ 'date_processed': datetimeutil.utc_now(), 'product': 'WaterWolf', }) self.refresh_index() mapping = self.api.get_mapping({ 'name': 'product', 'storage_mapping': { 'type': 'long' } }) assert_raises( BadArgumentError, self.api.test_mapping, mapping, )
class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() self.api = SuperSearchFields(config=self.config) self.api.get_fields = lambda: copy.deepcopy(SUPERSEARCH_FIELDS) def test_get_fields(self): results = self.api.get_fields() assert results == SUPERSEARCH_FIELDS @minimum_es_version('1.0') def test_get_missing_fields(self): config = self.get_base_config(es_index='socorro_integration_test_%W') fake_mappings = [ { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { # Add a bunch of unknown fields. 'field_z': { 'type': 'string' }, 'namespace1': { 'type': 'object', 'properties': { 'field_a': { 'type': 'string' }, 'field_b': { 'type': 'long' } } }, 'namespace2': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_b': { 'type': 'long' } } } } }, # Add a few known fields that should not appear. 'processed_crash': { 'type': 'object', 'properties': { 'signature': { 'type': 'string' }, 'product': { 'type': 'string' }, } } } } } }, { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { 'namespace1': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_d': { 'type': 'long' } } } } } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ 'field_z', 'namespace1.field_a', 'namespace1.field_b', 'namespace1.subspace1.field_d', 'namespace2.subspace1.field_b', ] assert missing_fields['hits'] == expected assert missing_fields['total'] == 5 finally: for index in indices: self.index_client.delete(index=index) def test_get_mapping(self): mapping = self.api.get_mapping() doctype = self.config.elasticsearch.elasticsearch_doctype assert doctype in mapping properties = mapping[doctype]['properties'] assert 'processed_crash' in properties assert 'raw_crash' in properties processed_crash = properties['processed_crash']['properties'] # Check in_database_name is used. assert 'os_name' in processed_crash assert 'platform' not in processed_crash # Those fields have no `storage_mapping`. assert 'fake_field' not in properties['raw_crash']['properties'] # Those fields have a `storage_mapping`. assert processed_crash['release_channel'] == {'type': 'string'} # Test nested objects. assert 'json_dump' in processed_crash assert 'properties' in processed_crash['json_dump'] assert 'write_combine_size' in processed_crash['json_dump'][ 'properties'] assert processed_crash['json_dump']['properties'][ 'write_combine_size'] == { 'type': 'long' } # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={ 'name': 'fake_field', 'storage_mapping': { 'type': 'long' } }) properties = mapping[doctype]['properties'] assert 'fake_field' in properties['raw_crash']['properties'] assert properties['raw_crash']['properties']['fake_field'][ 'type'] == 'long' def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() assert self.api.test_mapping(mapping) is None # Insert an invalid storage mapping. mapping = self.api.get_mapping({ 'name': 'fake_field', 'storage_mapping': { 'type': 'unkwown' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({ 'date_processed': datetimeutil.utc_now(), 'product': 'WaterWolf', }) self.refresh_index() mapping = self.api.get_mapping({ 'name': 'product', 'storage_mapping': { 'type': 'long' } }) with pytest.raises(BadArgumentError): self.api.test_mapping(mapping)
class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() self.api = SuperSearchFields(config=self.config) def test_get_fields(self): results = self.api.get_fields() eq_(results, SUPERSEARCH_FIELDS) def test_create_field(self): # Test with all parameters set. params = { "name": "plotfarm", "data_validation_type": "str", "default_value": None, "description": "a plotfarm like Lunix or Wondiws", "form_field_choices": ["lun", "won", "cam"], "has_full_version": True, "in_database_name": "os_name", "is_exposed": True, "is_returned": True, "is_mandatory": False, "query_type": "str", "namespace": "processed_crash", "permissions_needed": ["view_plotfarm"], "storage_mapping": {"type": "multi_field"}, } res = self.api.create_field(**params) ok_(res) field = self.connection.get( index=self.config.webapi.elasticsearch_default_index, doc_type="supersearch_fields", id="plotfarm" ) field = field["_source"] eq_(sorted(field.keys()), sorted(params.keys())) for key in field.keys(): eq_(field[key], params[key]) # Test default values. res = self.api.create_field( name="brand_new_field", in_database_name="brand_new_field", namespace="processed_crash" ) ok_(res) ok_( self.connection.get( index=self.config.webapi.elasticsearch_default_index, doc_type="supersearch_fields", id="brand_new_field", ) ) # Test errors. # `name` is missing. assert_raises(MissingArgumentError, self.api.create_field, in_database_name="something") # `in_database_name` is missing. assert_raises(MissingArgumentError, self.api.create_field, name="something") # Field already exists. assert_raises( InsertionError, self.api.create_field, name="product", in_database_name="product", namespace="processed_crash", ) # Test logging. res = self.api.create_field( name="what_a_field", in_database_name="what_a_field", namespace="processed_crash", storage_mapping='{"type": "long"}', ) ok_(res) self.api.config.logger.info.assert_called_with( 'elasticsearch mapping changed for field "%s", ' 'added new mapping "%s"', "what_a_field", {u"type": u"long"}, ) def test_update_field(self): # Let's create a field first. assert self.api.create_field( name="super_field", in_database_name="super_field", namespace="superspace", description="inaccurate description", permissions_needed=["view_nothing"], storage_mapping={"type": "boolean", "null_value": False}, ) # Now let's update that field a little. res = self.api.update_field( name="super_field", description="very accurate description", storage_mapping={"type": "long", "analyzer": "keyword"}, ) ok_(res) # Test logging. self.api.config.logger.info.assert_called_with( 'elasticsearch mapping changed for field "%s", ' 'was "%s", now "%s"', "super_field", {"type": "boolean", "null_value": False}, {"type": "long", "analyzer": "keyword"}, ) field = self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type="supersearch_fields", id="super_field" ) field = field["_source"] # Verify the changes were taken into account. eq_(field["description"], "very accurate description") eq_(field["storage_mapping"], {"type": "long", "analyzer": "keyword"}) # Verify other values did not change. eq_(field["permissions_needed"], ["view_nothing"]) eq_(field["in_database_name"], "super_field") eq_(field["namespace"], "superspace") # Test errors. assert_raises(MissingArgumentError, self.api.update_field) # `name` is missing assert_raises(ResourceNotFound, self.api.update_field, name="unkownfield") def test_delete_field(self): self.api.delete_field(name="product") ok_( self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type="supersearch_fields", id="signature", ) ) assert_raises( elasticsearch.exceptions.NotFoundError, self.connection.get, index=self.config.elasticsearch.elasticsearch_default_index, doc_type="supersearch_fields", id="product", ) @minimum_es_version("1.0") def test_get_missing_fields(self): config = self.get_mware_config(es_index="socorro_integration_test_%W") fake_mappings = [ { "mappings": { config.elasticsearch.elasticsearch_doctype: { "properties": { # Add a bunch of unknown fields. "field_z": {"type": "string"}, "namespace1": { "type": "object", "properties": {"field_a": {"type": "string"}, "field_b": {"type": "long"}}, }, "namespace2": { "type": "object", "properties": { "subspace1": {"type": "object", "properties": {"field_b": {"type": "long"}}} }, }, # Add a few known fields that should not appear. "processed_crash": { "type": "object", "properties": {"signature": {"type": "string"}, "product": {"type": "string"}}, }, } } } }, { "mappings": { config.elasticsearch.elasticsearch_doctype: { "properties": { "namespace1": { "type": "object", "properties": { "subspace1": {"type": "object", "properties": {"field_d": {"type": "long"}}} }, } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ "field_z", "namespace1.field_a", "namespace1.field_b", "namespace1.subspace1.field_d", "namespace2.subspace1.field_b", ] eq_(missing_fields["hits"], expected) eq_(missing_fields["total"], 5) finally: for index in indices: self.index_client.delete(index=index) def test_get_mapping(self): mapping = self.api.get_mapping()["mappings"] doctype = self.config.elasticsearch.elasticsearch_doctype ok_(doctype in mapping) properties = mapping[doctype]["properties"] ok_("processed_crash" in properties) ok_("raw_crash" in properties) processed_crash = properties["processed_crash"]["properties"] # Check in_database_name is used. ok_("os_name" in processed_crash) ok_("platform" not in processed_crash) # Those fields have no `storage_mapping`. ok_("fake_field" not in properties["raw_crash"]["properties"]) # Those fields have a `storage_mapping`. eq_(processed_crash["release_channel"], {"type": "string"}) # Test nested objects. ok_("json_dump" in processed_crash) ok_("properties" in processed_crash["json_dump"]) ok_("write_combine_size" in processed_crash["json_dump"]["properties"]) eq_(processed_crash["json_dump"]["properties"]["write_combine_size"], {"type": "long"}) # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={"name": "fake_field", "storage_mapping": {"type": "long"}})[ "mappings" ] properties = mapping[doctype]["properties"] ok_("fake_field" in properties["raw_crash"]["properties"]) eq_(properties["raw_crash"]["properties"]["fake_field"]["type"], "long") def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() ok_(self.api.test_mapping(mapping) is None) # Insert an invalid storage mapping. mapping = self.api.get_mapping({"name": "fake_field", "storage_mapping": {"type": "unkwown"}}) assert_raises(elasticsearch.exceptions.RequestError, self.api.test_mapping, mapping) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({"date_processed": datetimeutil.utc_now(), "product": "WaterWolf"}) self.refresh_index() mapping = self.api.get_mapping({"name": "product", "storage_mapping": {"type": "long"}}) # self.api.test_mapping(mapping) assert_raises(elasticsearch.exceptions.RequestError, self.api.test_mapping, mapping)
class IntegrationTestSuperSearchFields(ElasticsearchTestCase): """Test SuperSearchFields with an elasticsearch database containing fake data. """ def setUp(self): super(IntegrationTestSuperSearchFields, self).setUp() self.api = SuperSearchFields(config=self.config) def test_get_fields(self): results = self.api.get_fields() eq_(results, SUPERSEARCH_FIELDS) def test_create_field(self): # Test with all parameters set. params = { 'name': 'plotfarm', 'data_validation_type': 'str', 'default_value': None, 'description': 'a plotfarm like Lunix or Wondiws', 'form_field_choices': ['lun', 'won', 'cam'], 'has_full_version': False, 'in_database_name': 'os_name', 'is_exposed': True, 'is_returned': True, 'is_mandatory': False, 'query_type': 'str', 'namespace': 'processed_crash', 'permissions_needed': ['view_plotfarm'], 'storage_mapping': { "type": "keyword" }, } res = self.api.create_field(**params) ok_(res) field = self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='plotfarm', ) field = field['_source'] eq_(sorted(field.keys()), sorted(params.keys())) for key in field.keys(): eq_(field[key], params[key]) # Test default values. res = self.api.create_field( name='brand_new_field', in_database_name='brand_new_field', namespace='processed_crash', ) ok_(res) ok_( self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='brand_new_field', )) # Test errors. # `name` is missing. assert_raises( MissingArgumentError, self.api.create_field, in_database_name='something', ) # `in_database_name` is missing. assert_raises( MissingArgumentError, self.api.create_field, name='something', ) # Field already exists. assert_raises( BadArgumentError, self.api.create_field, name='product', in_database_name='product', namespace='processed_crash', ) # Test logging. res = self.api.create_field( name='what_a_field', in_database_name='what_a_field', namespace='processed_crash', storage_mapping='{"type": "long"}', ) ok_(res) self.api.config.logger.info.assert_called_with( 'elasticsearch mapping changed for field "%s", ' 'added new mapping "%s"', 'what_a_field', {u'type': u'long'}, ) def test_update_field(self): # Let's create a field first. assert self.api.create_field(name='super_field', in_database_name='super_field', namespace='superspace', description='inaccurate description', permissions_needed=['view_nothing'], storage_mapping={ 'type': 'boolean', 'null_value': False }) # Now let's update that field a little. res = self.api.update_field( name='super_field', description='very accurate description', storage_mapping={'type': 'long'}, ) ok_(res) # Test logging. self.api.config.logger.info.assert_called_with( 'Elasticsearch mapping changed for field "%s", ' 'was "%s", now "%s"', 'super_field', { 'type': 'boolean', 'null_value': False }, {'type': 'long'}, ) field = self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='super_field', ) field = field['_source'] # Verify the changes were taken into account. eq_(field['description'], 'very accurate description') eq_(field['storage_mapping'], {'type': 'long'}) # Verify other values did not change. eq_(field['permissions_needed'], ['view_nothing']) eq_(field['in_database_name'], 'super_field') eq_(field['namespace'], 'superspace') # Test errors. assert_raises( MissingArgumentError, self.api.update_field, ) # `name` is missing assert_raises( ResourceNotFound, self.api.update_field, name='unkownfield', ) def test_delete_field(self): self.api.delete_field(name='product') ok_( self.connection.get( index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='signature', )) assert_raises( elasticsearch.exceptions.NotFoundError, self.connection.get, index=self.config.elasticsearch.elasticsearch_default_index, doc_type='supersearch_fields', id='product', ) def test_get_missing_fields(self): config = self.get_base_config(es_index='socorro_integration_test_%W') fake_mappings = [ { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { # Add a bunch of unknown fields. 'field_z': { 'type': 'string' }, 'namespace1': { 'type': 'object', 'properties': { 'field_a': { 'type': 'string' }, 'field_b': { 'type': 'long' } } }, 'namespace2': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_b': { 'type': 'long' } } } } }, # Add a few known fields that should not appear. 'processed_crash': { 'type': 'object', 'properties': { 'signature': { 'type': 'string' }, 'product': { 'type': 'string' }, } } } } } }, { 'mappings': { config.elasticsearch.elasticsearch_doctype: { 'properties': { 'namespace1': { 'type': 'object', 'properties': { 'subspace1': { 'type': 'object', 'properties': { 'field_d': { 'type': 'long' } } } } } } } } }, ] now = datetimeutil.utc_now() indices = [] try: # Using "2" here means that an index will be missing, hence testing # that it swallows the subsequent error. for i in range(2): date = now - datetime.timedelta(weeks=i) index = date.strftime(config.elasticsearch.elasticsearch_index) mapping = fake_mappings[i % len(fake_mappings)] self.index_creator.create_index(index, mapping) indices.append(index) api = SuperSearchFields(config=config) missing_fields = api.get_missing_fields() expected = [ 'field_z', 'namespace1.field_a', 'namespace1.field_b', 'namespace1.subspace1.field_d', 'namespace2.subspace1.field_b', ] eq_(missing_fields['hits'], expected) eq_(missing_fields['total'], 5) finally: for index in indices: self.index_client.delete(index=index) def test_get_mapping(self): mapping = self.api.get_mapping() doctype = self.config.elasticsearch.elasticsearch_doctype ok_(doctype in mapping) properties = mapping[doctype]['properties'] ok_('processed_crash' in properties) ok_('raw_crash' in properties) processed_crash = properties['processed_crash']['properties'] # Check in_database_name is used. ok_('os_name' in processed_crash) ok_('platform' not in processed_crash) # Those fields have no `storage_mapping`. ok_('fake_field' not in properties['raw_crash']['properties']) # Those fields have a `storage_mapping`. eq_(processed_crash['signature'], { 'type': 'text', 'fields': { 'full': { 'type': 'keyword', } } }) # Test nested objects. ok_('json_dump' in processed_crash) ok_('properties' in processed_crash['json_dump']) ok_('write_combine_size' in processed_crash['json_dump']['properties']) eq_(processed_crash['json_dump']['properties']['write_combine_size'], {'type': 'long'}) # Test overwriting a field. mapping = self.api.get_mapping(overwrite_mapping={ 'name': 'fake_field', 'storage_mapping': { 'type': 'long' } }) properties = mapping[doctype]['properties'] ok_('fake_field' in properties['raw_crash']['properties']) eq_(properties['raw_crash']['properties']['fake_field']['type'], 'long') def test_test_mapping(self): """Much test. So meta. Wow test_test_. """ # First test a valid mapping. mapping = self.api.get_mapping() ok_(self.api.test_mapping(mapping) is None) # Insert an invalid storage mapping. mapping = self.api.get_mapping({ 'name': 'fake_field', 'storage_mapping': { 'type': 'unkwown' } }) assert_raises( BadArgumentError, self.api.test_mapping, mapping, ) # Test with a correct mapping but with data that cannot be indexed. self.index_crash({ 'date_processed': datetimeutil.utc_now(), 'product': 'WaterWolf', }) self.refresh_index() mapping = self.api.get_mapping({ 'name': 'product', 'storage_mapping': { 'type': 'long' } }) assert_raises( BadArgumentError, self.api.test_mapping, mapping, )