Python index_document Beispiele

Beispiel #1

Datei: test_index_utils.py Projekt: symroe/addok

def test_index_document():
    index_document(DOC.copy())
    assert DB.exists('d|xxxx')
    assert DB.type('d|xxxx') == b'hash'
    assert DB.exists('w|rue')
    assert b'd|xxxx' in DB.zrange('w|rue', 0, -1)
    assert DB.exists('w|des')
    assert DB.exists('w|lilas')
    assert DB.exists('w|andresy')
    assert DB.exists('w|un')  # Housenumber.
    assert DB.exists('p|rue')
    assert DB.exists('p|des')
    assert DB.exists('p|lilas')
    assert DB.exists('p|andresy')
    assert b'lilas' in DB.smembers('p|andresy')
    assert b'andresy' in DB.smembers('p|lilas')
    assert DB.exists('p|un')
    assert DB.exists('g|u09dgm7')
    assert b'd|xxxx' in DB.smembers('g|u09dgm7')
    assert DB.exists('n|lil')
    assert DB.exists('n|lila')
    assert DB.exists('n|and')
    assert b'andresy' in DB.smembers('n|and')
    assert DB.exists('n|andr')
    assert b'andresy' in DB.smembers('n|andr')
    assert DB.exists('n|andre')
    assert b'andresy' in DB.smembers('n|andre')
    assert DB.exists('n|andres')
    assert b'andresy' in DB.smembers('n|andres')
    assert b'lilas' in DB.smembers('n|lil')
    assert DB.exists('f|type|street')
    assert b'd|xxxx' in DB.smembers('f|type|street')
    assert DB.exists('f|type|housenumber')
    assert b'd|xxxx' in DB.smembers('f|type|housenumber')
    assert len(DB.keys()) == 20

Beispiel #2

Datei: test_index_utils.py Projekt: xlqian/addok

def test_index_document():
    index_document(DOC.copy())
    assert DB.exists('d|xxxx')
    assert DB.type('d|xxxx') == b'hash'
    assert DB.exists('w|rue')
    assert b'd|xxxx' in DB.zrange('w|rue', 0, -1)
    assert DB.exists('w|des')
    assert DB.exists('w|lilas')
    assert DB.exists('w|andresy')
    assert DB.exists('w|un')  # Housenumber.
    assert DB.exists('p|rue')
    assert DB.exists('p|des')
    assert DB.exists('p|lilas')
    assert DB.exists('p|andresy')
    assert b'lilas' in DB.smembers('p|andresy')
    assert b'andresy' in DB.smembers('p|lilas')
    assert DB.exists('p|un')
    assert DB.exists('g|u09dgm7')
    assert b'd|xxxx' in DB.smembers('g|u09dgm7')
    assert DB.exists('n|lil')
    assert DB.exists('n|lila')
    assert DB.exists('n|and')
    assert b'andresy' in DB.smembers('n|and')
    assert DB.exists('n|andr')
    assert b'andresy' in DB.smembers('n|andr')
    assert DB.exists('n|andre')
    assert b'andresy' in DB.smembers('n|andre')
    assert DB.exists('n|andres')
    assert b'andresy' in DB.smembers('n|andres')
    assert b'lilas' in DB.smembers('n|lil')
    assert DB.exists('f|type|street')
    assert b'd|xxxx' in DB.smembers('f|type|street')
    assert DB.exists('f|type|housenumber')
    assert b'd|xxxx' in DB.smembers('f|type|housenumber')
    assert len(DB.keys()) == 20

Beispiel #3

Datei: test_index_utils.py Projekt: symroe/addok

def test_field_with_only_non_alphanumeric_chars_is_not_indexed():
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Lilas',
        'city': '//'
    }
    index_document(doc)
    assert 'city' not in DB.hgetall('d|xxxx')

Beispiel #4

Datei: test_index_utils.py Projekt: xlqian/addok

def test_field_with_only_non_alphanumeric_chars_is_not_indexed():
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Lilas',
        'city': '//'
    }
    index_document(doc)
    assert 'city' not in DB.hgetall('d|xxxx')

Beispiel #5

Datei: test_index_utils.py Projekt: symroe/addok

def test_null_value_should_not_be_index(config):
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Port-Cergy',
        'city': ''
    }
    index_document(doc)
    assert 'city' not in DB.hgetall('d|xxxx')

Beispiel #6

Datei: test_index_utils.py Projekt: xlqian/addok

def test_null_value_should_not_be_index(config):
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Port-Cergy',
        'city': ''
    }
    index_document(doc)
    assert 'city' not in DB.hgetall('d|xxxx')

Beispiel #7

Datei: test_index_utils.py Projekt: symroe/addok

def test_deindex_document_should_not_affect_other_docs():
    DOC2 = {
        'id': 'xxxx2',
        'type': 'street',
        'name': 'rue des Lilas',
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565',
        'housenumbers': {
            '1': {
                'lat': '48.325451',  # Same geohash as DOC.
                'lon': '2.25651'
            }
        }
    }
    index_document(DOC.copy())
    index_document(DOC2)
    deindex_document(DOC['id'])
    assert not DB.exists('d|xxxx')
    assert b'd|xxxx' not in DB.zrange('w|rue', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|des', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|lilas', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|un', 0, -1)
    assert DB.exists('g|u09dgm7')
    assert b'd|xxxx' not in DB.smembers('g|u09dgm7')
    assert DB.exists('w|des')
    assert DB.exists('w|lilas')
    assert DB.exists('w|un')  # Housenumber.
    assert DB.exists('p|rue')
    assert b'd|xxxx2' in DB.zrange('w|rue', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|des', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|lilas', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|un', 0, -1)
    assert b'd|xxxx2' in DB.smembers('g|u09dgm7')
    assert b'd|xxxx2' in DB.smembers('g|u0g08g7')
    assert DB.exists('p|des')
    assert DB.exists('p|lilas')
    assert DB.exists('p|un')
    assert not DB.exists('n|and')
    assert not DB.exists('n|andr')
    assert not DB.exists('n|andre')
    assert not DB.exists('n|andres')
    assert DB.exists('n|par')
    assert DB.exists('n|pari')
    assert DB.exists('n|lil')
    assert DB.exists('n|lila')
    assert b'lilas' in DB.smembers('n|lil')
    assert b'lilas' in DB.smembers('n|lila')
    assert DB.exists('f|type|street')
    assert b'd|xxxx2' in DB.smembers('f|type|street')
    assert DB.exists('f|type|housenumber')
    assert b'd|xxxx2' in DB.smembers('f|type|housenumber')
    assert len(DB.keys()) == 19

Beispiel #8

Datei: test_index_utils.py Projekt: xlqian/addok

def test_deindex_document_should_not_affect_other_docs():
    DOC2 = {
        'id': 'xxxx2',
        'type': 'street',
        'name': 'rue des Lilas',
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565',
        'housenumbers': {
            '1': {
                'lat': '48.325451',  # Same geohash as DOC.
                'lon': '2.25651'
            }
        }
    }
    index_document(DOC.copy())
    index_document(DOC2)
    deindex_document(DOC['id'])
    assert not DB.exists('d|xxxx')
    assert b'd|xxxx' not in DB.zrange('w|rue', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|des', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|lilas', 0, -1)
    assert b'd|xxxx' not in DB.zrange('w|un', 0, -1)
    assert DB.exists('g|u09dgm7')
    assert b'd|xxxx' not in DB.smembers('g|u09dgm7')
    assert DB.exists('w|des')
    assert DB.exists('w|lilas')
    assert DB.exists('w|un')  # Housenumber.
    assert DB.exists('p|rue')
    assert b'd|xxxx2' in DB.zrange('w|rue', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|des', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|lilas', 0, -1)
    assert b'd|xxxx2' in DB.zrange('w|un', 0, -1)
    assert b'd|xxxx2' in DB.smembers('g|u09dgm7')
    assert b'd|xxxx2' in DB.smembers('g|u0g08g7')
    assert DB.exists('p|des')
    assert DB.exists('p|lilas')
    assert DB.exists('p|un')
    assert not DB.exists('n|and')
    assert not DB.exists('n|andr')
    assert not DB.exists('n|andre')
    assert not DB.exists('n|andres')
    assert DB.exists('n|par')
    assert DB.exists('n|pari')
    assert DB.exists('n|lil')
    assert DB.exists('n|lila')
    assert b'lilas' in DB.smembers('n|lil')
    assert b'lilas' in DB.smembers('n|lila')
    assert DB.exists('f|type|street')
    assert b'd|xxxx2' in DB.smembers('f|type|street')
    assert DB.exists('f|type|housenumber')
    assert b'd|xxxx2' in DB.smembers('f|type|housenumber')
    assert len(DB.keys()) == 19

Beispiel #9

Datei: test_index_utils.py Projekt: xlqian/addok

def test_allow_list_values():
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': ['Vernou-la-Celle-sur-Seine', 'Vernou'],
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565'
    }
    index_document(doc)
    assert DB.zscore('w|vernou', 'd|xxxx') == 4
    assert DB.zscore('w|celle', 'd|xxxx') == 4 / 5

Beispiel #10

Datei: test_index_utils.py Projekt: symroe/addok

def test_allow_list_values():
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': ['Vernou-la-Celle-sur-Seine', 'Vernou'],
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565'
    }
    index_document(doc)
    assert DB.zscore('w|vernou', 'd|xxxx') == 4
    assert DB.zscore('w|celle', 'd|xxxx') == 4 / 5

Beispiel #11

Datei: test_utils.py Projekt: addok/addok-france

def test_index_housenumber_uses_housenumber_preprocessors(config):
    doc = {
        "id": "xxxx",
        "type": "street",
        "name": "rue des Lilas",
        "city": "Paris",
        "lat": "49.32545",
        "lon": "4.2565",
        "housenumbers": {"1 bis": {"lat": "48.325451", "lon": "2.25651"}},
    }
    index_document(doc)
    index = DB.hgetall("d|xxxx")
    assert index[b"h|1b"] == b"1 bis|48.325451|2.25651"

Beispiel #12

Datei: test_index_utils.py Projekt: xlqian/addok

def test_doc_with_null_value_should_not_be_index_if_not_allowed(config):
    config.FIELDS = [
        {'key': 'name', 'null': False},
        {'key': 'city'},
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': '',
        'city': 'Cergy'
    }
    index_document(doc)
    assert not DB.exists('d|xxxx')

Beispiel #13

Datei: test_index_utils.py Projekt: xlqian/addok

def test_deindex_document_should_deindex_list_values():
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': ['Vernou-la-Celle-sur-Seine', 'Vernou'],
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565'
    }
    index_document(doc)
    deindex_document(doc['id'])
    assert not DB.exists('d|xxxx')
    assert not DB.exists('w|vernou')
    assert not DB.exists('w|celle')
    assert len(DB.keys()) == 0

Beispiel #14

Datei: test_index_utils.py Projekt: symroe/addok

def test_deindex_document_should_deindex_list_values():
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': ['Vernou-la-Celle-sur-Seine', 'Vernou'],
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565'
    }
    index_document(doc)
    deindex_document(doc['id'])
    assert not DB.exists('d|xxxx')
    assert not DB.exists('w|vernou')
    assert not DB.exists('w|celle')
    assert len(DB.keys()) == 0

Beispiel #15

Datei: test_index_utils.py Projekt: xlqian/addok

def test_should_be_possible_to_override_boost_with_callable(config):
    config.FIELDS = [
        {'key': 'name', 'boost': lambda doc: 5},
        {'key': 'city'},
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Lilas',
        'city': 'Cergy'
    }
    index_document(doc)
    assert DB.exists('d|xxxx')
    assert DB.zscore('w|lilas', 'd|xxxx') == 5
    assert DB.zscore('w|cergy', 'd|xxxx') == 1

Beispiel #16

Datei: test_index_utils.py Projekt: xlqian/addok

def test_should_be_possible_to_define_fields_from_config(config):
    config.FIELDS = [
        {'key': 'custom'},
        {'key': 'special'},
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'custom': 'rue',
        'special': 'Lilas',
        'thisone': 'is not indexed',
    }
    index_document(doc)
    assert DB.exists('d|xxxx')
    assert DB.exists('w|lilas')
    assert DB.exists('w|rue')
    assert not DB.exists('w|indexed')

Beispiel #17

Datei: test_index.py Projekt: addok/addok-trigrams

def test_deindex_document_should_deindex():
    index_document(DOC.copy())
    deindex_document(DOC['id'])
    assert not DB.exists('d|xxxx')
    assert not DB.exists('w|des')
    assert not DB.exists('w|lil')
    assert not DB.exists('w|ila')
    assert not DB.exists('w|las')
    assert not DB.exists('w|and')
    assert not DB.exists('w|ndr')
    assert not DB.exists('w|dre')
    assert not DB.exists('w|res')
    assert not DB.exists('w|esy')
    assert not DB.exists('w|un')  # Housenumber.
    assert not DB.exists('g|u09dgm7')
    assert not DB.exists('f|type|street')
    assert not DB.exists('f|type|housenumber')
    assert len(DB.keys()) == 0

Beispiel #18

Datei: test_index.py Projekt: addok/addok-trigrams

def test_index_housenumber_uses_housenumber_preprocessors():
    # By default it glues ordinal to number
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': 'rue des Lilas',
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565',
        'housenumbers': {
            '1 bis': {
                'lat': '48.325451',
                'lon': '2.25651'
            }
        }
    }
    index_document(doc)
    index = DB.hgetall('d|xxxx')
    assert index[b'h|1bis'] == b'1 bis|48.325451|2.25651'

Beispiel #19

Datei: test_index_utils.py Projekt: symroe/addok

def test_deindex_document_should_deindex():
    index_document(DOC.copy())
    deindex_document(DOC['id'])
    assert not DB.exists('d|xxxx')
    assert not DB.exists('w|de')
    assert not DB.exists('w|lilas')
    assert not DB.exists('w|un')  # Housenumber.
    assert not DB.exists('p|rue')
    assert not DB.exists('p|des')
    assert not DB.exists('p|lilas')
    assert not DB.exists('p|un')
    assert not DB.exists('g|u09dgm7')
    assert not DB.exists('n|lil')
    assert not DB.exists('n|and')
    assert not DB.exists('n|andr')
    assert not DB.exists('n|andre')
    assert not DB.exists('n|andres')
    assert not DB.exists('f|type|street')
    assert len(DB.keys()) == 0

Beispiel #20

Datei: test_index_utils.py Projekt: symroe/addok

def test_doc_with_null_value_should_not_be_index_if_not_allowed(config):
    config.FIELDS = [
        {
            'key': 'name',
            'null': False
        },
        {
            'key': 'city'
        },
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': '',
        'city': 'Cergy'
    }
    index_document(doc)
    assert not DB.exists('d|xxxx')

Beispiel #21

Datei: test_index_utils.py Projekt: symroe/addok

def test_should_be_possible_to_override_boost_with_callable(config):
    config.FIELDS = [
        {
            'key': 'name',
            'boost': lambda doc: 5
        },
        {
            'key': 'city'
        },
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': 'Lilas',
        'city': 'Cergy'
    }
    index_document(doc)
    assert DB.exists('d|xxxx')
    assert DB.zscore('w|lilas', 'd|xxxx') == 5
    assert DB.zscore('w|cergy', 'd|xxxx') == 1

Beispiel #22

Datei: test_index_utils.py Projekt: xlqian/addok

def test_index_should_join_housenumbers_payload_fields(config):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['key', 'one']
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': 'rue des Lilas',
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565',
        'housenumbers': {
            '1 bis': {
                'lat': '48.325451',
                'lon': '2.25651',
                'key': 'myvalue',
                'thisone': 'no',
                'one': 'two',
            }
        }
    }
    index_document(doc)
    index = DB.hgetall('d|xxxx')
    assert index[b'h|1bis'] == b'1 bis|48.325451|2.25651|myvalue|two'

Beispiel #23

Datei: test_index_utils.py Projekt: symroe/addok

def test_should_be_possible_to_define_fields_from_config(config):
    config.FIELDS = [
        {
            'key': 'custom'
        },
        {
            'key': 'special'
        },
    ]
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'custom': 'rue',
        'special': 'Lilas',
        'thisone': 'is not indexed',
    }
    index_document(doc)
    assert DB.exists('d|xxxx')
    assert DB.exists('w|lilas')
    assert DB.exists('w|rue')
    assert not DB.exists('w|indexed')

Beispiel #24

Datei: test_index_utils.py Projekt: symroe/addok

def test_index_should_join_housenumbers_payload_fields(config):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['key', 'one']
    doc = {
        'id': 'xxxx',
        'type': 'street',
        'name': 'rue des Lilas',
        'city': 'Paris',
        'lat': '49.32545',
        'lon': '4.2565',
        'housenumbers': {
            '1 bis': {
                'lat': '48.325451',
                'lon': '2.25651',
                'key': 'myvalue',
                'thisone': 'no',
                'one': 'two',
            }
        }
    }
    index_document(doc)
    index = DB.hgetall('d|xxxx')
    assert index[b'h|1bis'] == b'1 bis|48.325451|2.25651|myvalue|two'

Beispiel #25

Datei: test_index_utils.py Projekt: xlqian/addok

def test_create_edge_ngrams(config):
    config.MIN_EDGE_NGRAMS = 2
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': '28 Lilas',  # 28 should not appear in ngrams
        'city': 'Paris'
    }
    index_document(doc, update_ngrams=False)
    assert not DB.exists('n|li')
    assert not DB.exists('n|lil')
    assert not DB.exists('n|lila')
    assert not DB.exists('n|pa')
    assert not DB.exists('n|par')
    assert not DB.exists('n|pari')
    create_edge_ngrams()
    assert DB.exists('n|li')
    assert DB.exists('n|lil')
    assert DB.exists('n|pa')
    assert DB.exists('n|par')
    assert not DB.exists('n|28')
    assert len(DB.keys()) == 14

Beispiel #26

Datei: test_index_utils.py Projekt: symroe/addok

def test_create_edge_ngrams(config):
    config.MIN_EDGE_NGRAMS = 2
    config.INDEX_EDGE_NGRAMS = False
    doc = {
        'id': 'xxxx',
        'lat': '49.32545',
        'lon': '4.2565',
        'name': '28 Lilas',  # 28 should not appear in ngrams
        'city': 'Paris'
    }
    index_document(doc)
    assert not DB.exists('n|li')
    assert not DB.exists('n|lil')
    assert not DB.exists('n|lila')
    assert not DB.exists('n|pa')
    assert not DB.exists('n|par')
    assert not DB.exists('n|pari')
    create_edge_ngrams()
    assert DB.exists('n|li')
    assert DB.exists('n|lil')
    assert DB.exists('n|pa')
    assert DB.exists('n|par')
    assert not DB.exists('n|28')
    assert len(DB.keys()) == 14

Beispiel #27

Datei: batch.py Projekt: symroe/addok

def process(doc):
    if doc.get('_action') in ['delete', 'update']:
        deindex_document(doc['id'])
    if doc.get('_action') in ['index', 'update', None]:
        index_document(doc)

Beispiel #28

Datei: batch.py Projekt: ernest33/addok

def process(doc):
    if doc.get('_action') in ['delete', 'update']:
        deindex_document(doc['id'])
    if doc.get('_action') in ['index', 'update', None]:
        index_document(doc)

Beispiel #29

Datei: pytest.py Projekt: eric-pommereau/addok

 def index(self):
     from addok.helpers.index import index_document
     index_document(self)

Beispiel #30

 def index(self):
     from addok.helpers.index import index_document
     index_document(self)