Beispiel #1
0
def test_should_give_priority_to_housenumber_if_match(housenumber):
    housenumber.update(name='rue des Berges')
    results = search('rue des berges')
    assert not results[0].housenumber
    results = search('11 rue des berges')
    assert results[0].housenumber == '11'
    assert results[0].type == 'housenumber'
Beispiel #2
0
def test_housenumber_id_is_used_when_given(factory):
    factory(name="rue de Paris", type="street", id="123",
            housenumbers={'1': {'lat': '48.325', 'lon': '2.256', 'id': 'abc'}})
    results = search("rue de paris")
    assert results[0].id == '123'
    results = search("1 rue de paris")
    assert results[0].id == 'abc'
Beispiel #3
0
def test_return_housenumber_if_number_included_in_bigger_one(factory):
    factory(name='rue 1814',
            housenumbers={'8': {'lat': '48.3254', 'lon': '2.256'}})
    results = search('rue 1814')
    assert not results[0].housenumber
    results = search('8 rue 1814')
    assert results[0].housenumber == '8'
Beispiel #4
0
def test_id_is_overwritten_when_given_in_housenumber_payload(config, factory):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['id']
    factory(name="rue de Paris", type="street", id="123",
            housenumbers={'1': {'lat': '48.325', 'lon': '2.256', 'id': 'abc'}})
    results = search("rue de paris")
    assert results[0].id == '123'
    results = search("1 rue de paris")
    assert results[0].id == 'abc'
Beispiel #5
0
def test_unknown_key_in_housenumber_payload_does_not_fail(config, factory):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['xxxyyy']
    factory(name="rue de Paris", type="street", id="123", postcode="12345",
            housenumbers={'1': {'lat': '48.325', 'lon': '2.256'}})
    results = search("rue de paris")
    assert results[0].id == '123'
    results = search("1 rue de paris")
    assert results[0].id == '123'
Beispiel #6
0
def test_process_should_update_if_action_is_given(factory):
    doc = factory(name="rue de l'avoine")
    assert search("rue")
    doc["_action"] = "update"
    doc["name"] = "avenue de l'avoine"
    process(doc)
    assert search("avenue")
    assert not search("rue")
Beispiel #7
0
def test_should_match_name(street):
    assert not search('Conflans')
    street.update(name='Conflans')
    results = search('Conflans')
    assert results
    result = results[0]
    assert result.name == 'Conflans'
    assert result.id == street['id']
Beispiel #8
0
def test_housenumbers_payload_fields_are_exported(config, factory):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['key']
    factory(name="rue de Paris", type="street", id="123",
            housenumbers={'1': {'lat': '48.32', 'lon': '2.25', 'key': 'abc'}})
    results = search("rue de paris")
    assert results[0].key == ''
    results = search("1 rue de paris")
    assert results[0].key == 'abc'
Beispiel #9
0
def test_postcode_is_overwritten_when_in_housenumber_payload(config, factory):
    config.HOUSENUMBERS_PAYLOAD_FIELDS = ['postcode']
    factory(name="rue de Paris", type="street", id="123", postcode="12345",
            housenumbers={'1': {'lat': '48.325', 'lon': '2.256',
                                'postcode': '54321'}})
    results = search("rue de paris")
    assert results[0].postcode == '12345'
    results = search("1 rue de paris")
    assert results[0].postcode == '54321'
Beispiel #10
0
def test_should_give_priority_to_best_match3(street, factory):
    street.update(name="rue de Lille", city="Douai")
    other = factory(name="rue de Douai", city="Lille")
    results = search("rue de lille douai")
    assert len(results) == 2
    assert results[0].id == street['id']
    results = search("rue de douai lille")
    assert len(results) == 2
    assert results[0].id == other['id']
Beispiel #11
0
def test_housenumber_are_not_computed_if_another_type_is_asked(factory):
    factory(name="rue de Bamako", type="street",
            housenumbers={'11': {'lat': '48.3254', 'lon': '2.256'}})

    results = search("11 rue de bamako")
    assert len(results) == 1
    assert results[0].type == "housenumber"

    results = search("11 rue de bamako", type="housenumber")
    assert len(results) == 1
    assert results[0].type == "housenumber"

    results = search("11 rue de bamako", type="street")
    assert len(results) == 1
    assert results[0].type == "street"
Beispiel #12
0
 def get(self):
     query = self.request.args.get('q', '')
     if not query:
         return Response('Missing query', status=400)
     try:
         limit = int(self.request.args.get('limit'))
     except (ValueError, TypeError):
         limit = 5
     try:
         autocomplete = int(self.request.args.get('autocomplete')) == 1
     except (ValueError, TypeError):
         autocomplete = True
     try:
         lat = float(self.request.args.get('lat'))
         lon = float(self.request.args.get('lon',
                     self.request.args.get('lng',
                     self.request.args.get('long'))))
         center = [lat, lon]
     except (ValueError, TypeError):
         lat = None
         lon = None
         center = None
     filters = self.match_filters()
     results = search(query, limit=limit, autocomplete=autocomplete,
                      lat=lat, lon=lon, **filters)
     if not results:
         log_notfound(query)
     log_query(query, results)
     return self.to_geojson(results, query=query, filters=filters,
                            center=center, limit=limit)
Beispiel #13
0
 def process_row(self, req, row, filters, columns):
     # We don't want None in a join.
     q = ' '.join([row[k] or '' for k in columns])
     filters = self.match_row_filters(row, filters)
     lat_column = req.get_param('lat')
     lon_column = req.get_param('lon')
     if lon_column and lat_column:
         lat = row.get(lat_column)
         lon = row.get(lon_column)
         if lat and lon:
             filters['lat'] = float(lat)
             filters['lon'] = float(lon)
     results = search(q, autocomplete=False, limit=1, **filters)
     log_query(q, results)
     if results:
         result = results[0]
         row.update({
             'latitude': result.lat,
             'longitude': result.lon,
             'result_label': str(result),
             'result_score': round(result.score, 2),
             'result_type': result.type,
             'result_id': result.id,
             'result_housenumber': result.housenumber,
         })
         self.add_extra_fields(row, result)
     else:
         log_notfound(q)
Beispiel #14
0
def test_closer_result_should_be_first_for_same_score(factory):
    expected = factory(name='rue de paris', city='Cergy', lat=48.1, lon=2.2)
    factory(name='rue de paris', city='Perpète', lat=-48.1, lon=-2.2)
    factory(name='rue de paris', city='Loin', lat=8.1, lon=42.2)
    results = search('rue de la monnaie', lat=48.1, lon=2.2)
    assert len(results) == 3
    assert results[0].id == expected['id']
Beispiel #15
0
def test_search_can_be_filtered(factory):
    street = factory(name="rue de Paris", type="street")
    city = factory(name="Paris", type="city")
    results = search("paris", type="street")
    ids = [r.id for r in results]
    assert street['id'] in ids
    assert city['id'] not in ids
Beispiel #16
0
def test_housenumber_type_can_be_filtered(factory):
    street_without_housenumber = factory(name="avenue de Paris", type="street")
    street_with_housenumber = factory(name="rue de Paris", type="street",
                                      housenumbers={'11': {'lat': '48.3254',
                                                           'lon': '2.256'}})
    results = search("paris", type="housenumber")
    ids = [r.id for r in results]
    assert street_with_housenumber['id'] in ids
    assert street_without_housenumber['id'] not in ids
Beispiel #17
0
def test_allow_to_set_result_values(factory):
    factory(name="porte des lilas", type="street", id="456")
    results = search("porte des lilas")
    result = results[0]
    result.name = "blah"
    result.score = 22
    # Plugins may need that.
    assert result.name == "blah"
    assert result.score == 22
Beispiel #18
0
def test_autocomplete_should_give_priority_to_nearby(factory, monkeypatch):
    monkeypatch.setattr('addok.config.config.BUCKET_LIMIT', 3)
    monkeypatch.setattr('addok.core.Search.SMALL_BUCKET_LIMIT', 2)
    expected = factory(name='Le Bourg', lat=48.1, lon=2.2, importance=0.09)
    factory(name='Le Bourg', lat=-48.1, lon=-2.2, importance=0.1)
    factory(name='Le Bourg', lat=8.1, lon=42.2, importance=0.1)
    factory(name='Le Bourg', lat=10, lon=20, importance=0.1)
    results = search('bou', lat=48.1, lon=2.2, limit=3)
    assert len(results) == 3
    ids = [r.id for r in results]
    assert expected['id'] in ids
Beispiel #19
0
def test_nearby_should_be_included_even_in_overflow(factory, monkeypatch):
    monkeypatch.setattr('addok.config.BUCKET_LIMIT', 3)
    monkeypatch.setattr('addok.core.Search.SMALL_BUCKET_LIMIT', 2)
    expected = factory(name='Le Bourg', lat=48.1, lon=2.2, importance=0.09)
    factory(name='Le Bourg', lat=-48.1, lon=-2.2, importance=0.1)
    factory(name='Le Bourg', lat=8.1, lon=42.2, importance=0.1)
    factory(name='Le Bourg', lat=10, lon=20, importance=0.1)
    results = search('bourg', lat=48.1, lon=2.2, limit=3, verbose=True)
    assert len(results) == 3
    ids = [r.id for r in results]
    assert expected['id'] in ids
Beispiel #20
0
def test_found_term_is_not_autocompleted_if_enough_results(factory,
                                                           monkeypatch):
    monkeypatch.setattr('addok.config.COMMON_THRESHOLD', 3)
    monkeypatch.setattr('addok.config.BUCKET_LIMIT', 3)
    montagne = factory(name="rue de la montagne", city="Vitry")
    factory(name="rue du mont", city="Vitry")
    factory(name="rue du mont", city="Paris")
    factory(name="rue du mont", city="Lille")
    results = search('rue mont', limit=2, autocomplete=True)
    ids = [r.id for r in results]
    assert len(ids) == 2
    assert montagne['id'] not in ids
Beispiel #21
0
def test_should_return_results_if_only_common_terms(factory, monkeypatch):
    monkeypatch.setattr('addok.config.config.COMMON_THRESHOLD', 3)
    monkeypatch.setattr('addok.config.config.BUCKET_LIMIT', 3)
    street1 = factory(name="rue de la monnaie", city="Vitry")
    street2 = factory(name="rue de la monnaie", city="Paris")
    street3 = factory(name="rue de la monnaie", city="Condom")
    street4 = factory(name="La monnaye", city="Saint-Loup-Cammas")
    results = search('rue de la monnaie')
    ids = [r.id for r in results]
    assert street1['id'] in ids
    assert street2['id'] in ids
    assert street3['id'] in ids
    assert street4['id'] not in ids
Beispiel #22
0
def test_config_make_labels_is_used_if_defined(config, factory):

    def make_labels(result):
        if result.name == "porte des lilas":
            return ['areallybadlabel']
        return [result.name]

    config.MAKE_LABELS = make_labels
    factory(name="porte des lilas", type="street", id="456", importance=1)
    factory(name="porte des Lilas", type="street", id="123")
    results = search("porte des lilas")
    assert results[0].id == "123"
    assert results[0].score > 0.9
    assert results[1].score > 0.1
Beispiel #23
0
 def on_get(self, req, resp, **kwargs):
     query = req.get_param('q')
     if not query:
         raise falcon.HTTPBadRequest('Missing query', 'Missing query')
     limit = req.get_param_as_int('limit') or 5  # use config
     autocomplete = req.get_param_as_bool('autocomplete')
     lon, lat = self.parse_lon_lat(req)
     center = None
     if lon and lat:
         center = (lon, lat)
     filters = self.match_filters(req)
     results = search(query, limit=limit, autocomplete=autocomplete,
                      lat=lat, lon=lon, **filters)
     if not results:
         log_notfound(query)
     log_query(query, results)
     self.to_geojson(req, resp, results, query=query, filters=filters,
                     center=center, limit=limit)
Beispiel #24
0
def test_does_not_fail_without_usable_tokens(street):
    assert not search('./.$*')
Beispiel #25
0
def test_fuzzy_should_work_with_inversion(city):
    city.update(name="Andrésy")
    assert search('andreys')
Beispiel #26
0
def test_process_should_deindex_if_action_is_given(factory):
    doc = factory(name="Mélicocq")
    assert search("Mélicoq")
    process_documents(json.dumps({"_action": "delete", "_id": doc["_id"]}))
    assert not search("Mélicoq")
Beispiel #27
0
def test_should_give_priority_to_best_match(street, city):
    street.update(name="rue d'Andrésy")
    city.update(name='Andrésy')
    results = search('andresy')
    assert results[0].id == city['id']
def test_should_be_fuzzy(city):
    city.update(name="Andrésy")
    assert search('antresy')
    assert search('antresu')
def test_synonyms_should_be_replaced(street, config):
    config.MIN_SCORE = 0
    config.SYNONYMS = {'bd': 'boulevard'}
    street.update(name='boulevard des Fleurs')
    assert search('bd')
Beispiel #30
0
def test_should_compare_with_multiple_values(city, factory):
    city.update(name=["Vernou-la-Celle-sur-Seine", "Vernou"])
    factory(name="Vernou", type="city")
    results = search("vernou")
    assert len(results) == 2
    assert results[0].score == results[1].score
Beispiel #31
0
def multiple_search(queries, **args):
    if len(queries) > 0:
        return max([search(query, **args) for query in queries],
                   key=lambda x: x and len(x) > 0 and x[0].score or 0)
    else:
        return []
Beispiel #32
0
def test_should_give_priority_to_best_match2(street, factory):
    street.update(name="rue d'Andrésy", city="Conflans")
    factory(name="rue de Conflans", city="Andrésy")
    results = search("rue andresy")
    assert len(results) == 2
    assert results[0].id == street['id']
Beispiel #33
0
def test_process_should_index_by_default(factory):
    doc = factory(skip_index=True, name="Melicocq")
    assert not search("Mélicocq")
    process(doc)
    assert search("Melicocq")
Beispiel #34
0
def test_should_do_autocomplete_on_last_term(street):
    street.update(name='rue de Wambrechies', city="Bondues")
    assert search('avenue wambre', autocomplete=True)
    assert not search('wambre avenue', autocomplete=True)
Beispiel #35
0
def test_fuzzy_should_match_with_removal(city):
    city.update(name="Andrésy")
    assert search('andressy')
Beispiel #36
0
def test_fuzzy_should_work_with_inversion(city):
    city.update(name="Andrésy")
    assert search('andreys')
Beispiel #37
0
def test_found_term_is_autocompleted_if_missing_results(factory, config):
    config.COMMON_THRESHOLD = 3
    config.BUCKET_MAX = 3
    factory(name="rue de la montagne", city="Vitry")
    factory(name="rue du mont", city="Vitry")
    assert len(search('rue mont', autocomplete=True)) == 2
Beispiel #38
0
def test_should_give_priority_to_best_match(street, city):
    street.update(name="rue d'Andrésy")
    city.update(name='Andrésy')
    results = search('andresy')
    assert results[0].id == city['id']
Beispiel #39
0
def test_should_be_fuzzy_of_1_by_default(city):
    city.update(name="Andrésy")
    assert search('antresy')
    assert not search('antresu')
Beispiel #40
0
def test_should_match_name_without_accent(street):
    assert not search('andresy')
    street.update(name='Andrésy')
    assert search('andresy')
def test_should_not_return_housenumber_if_number_is_also_in_name(housenumber):
    housenumber.update(name='rue du 11 Novembre')
    results = search('rue du 11 novembre')
    assert not results[0].housenumber
    results = search('11 rue du 11 novembre')
    assert results[0].housenumber == '11'
Beispiel #42
0
def test_score_is_not_greater_than_one(factory):
    factory(name='rue de paris', importance=1)
    results = search('rue de paris')
    assert len(results) == 1
    assert results[0].score == 1
def test_not_found_term_is_autocompleted(factory, config):
    config.COMMON_THRESHOLD = 3
    config.BUCKET_LIMIT = 3
    factory(name="rue de la monnaie", city="Vitry")
    assert search('rue de la mon')
Beispiel #44
0
def test_document_without_name_should_not_be_indexed(factory):
    doc = factory(skip_index=True, city="Montceau-les-Mines")
    del doc['name']
    doc.index()
    assert not search('Montceau-les-Mines')
Beispiel #45
0
def test_process_should_index_by_default(factory):
    doc = factory(skip_index=True, name="Melicocq")
    assert not search("Mélicocq")
    process_documents(json.dumps(doc.copy()))
    assert search("Melicocq")
Beispiel #46
0
def test_found_term_is_autocompleted_if_missing_results(factory, monkeypatch):
    monkeypatch.setattr('addok.config.config.COMMON_THRESHOLD', 3)
    monkeypatch.setattr('addok.config.config.BUCKET_MAX', 3)
    factory(name="rue de la montagne", city="Vitry")
    factory(name="rue du mont", city="Vitry")
    assert len(search('rue mont', autocomplete=True)) == 2
Beispiel #47
0
def test_importance_should_be_minored_if_geohash(factory, config):
    factory(name="rue descartes", lon=2.2, lat=48.1, importance=1)
    results = search('rue descartes')
    assert results[0]._scores['importance'][0] == 0.1
    results = search('rue descartes', lon=2.2, lat=48.1)
    assert results[0]._scores['importance'][0] == 0.010000000000000002
Beispiel #48
0
def test_not_found_term_is_autocompleted(factory, monkeypatch):
    monkeypatch.setattr('addok.config.config.COMMON_THRESHOLD', 3)
    monkeypatch.setattr('addok.config.config.BUCKET_MAX', 3)
    factory(name="rue de la monnaie", city="Vitry")
    assert search('rue de la mon')
Beispiel #49
0
def test_should_give_priority_to_best_match2(street, factory):
    street.update(name="rue d'Andrésy", city="Conflans")
    factory(name="rue de Conflans", city="Andrésy")
    results = search("rue andresy")
    assert len(results) == 2
    assert results[0].id == street['id']
Beispiel #50
0
def test_should_match_name_case_insensitive(street):
    assert not search('conflans')
    street.update(name='Conflans')
    assert search('conflans')
Beispiel #51
0
def test_should_be_fuzzy(city):
    city.update(name="Andrésy")
    assert search('antresy')
    assert search('antresu')
Beispiel #52
0
def test_synonyms_should_be_replaced(street, monkeypatch):
    monkeypatch.setattr('addok.helpers.text.SYNONYMS', {'bd': 'boulevard'})
    street.update(name='boulevard des Fleurs')
    assert search('bd')
Beispiel #53
0
def test_fuzzy_should_match_with_removal(city):
    city.update(name="Andrésy")
    assert search('andressy')
Beispiel #54
0
def search2steps(config, query1, queries2, autocomplete, limit, **filters):
    # Fetch the join value
    join_value = threshold = results = None

    # Run step 1 query
    results1 = search2steps_step1(config, query1,
                                  config.SEARCH_2_STEPS_STEP1_LIMIT, **filters)
    if len(queries2) == 0:
        ret = results1[0:limit]
        results_full = search(query1,
                              limit=limit,
                              autocomplete=autocomplete,
                              **filters)

    else:
        ret = []
        if results1:
            params_steps_2 = []
            # Collect step 1 results
            for result in results1:
                query_step_1 = result.__getattr__(
                    config.SEARCH_2_STEPS_PIVOT_REWRITE)

                if config.SEARCH_2_STEPS_PIVOT_FILTER in filters and filters[
                        config.SEARCH_2_STEPS_PIVOT_FILTER]:
                    join_value = filters[config.SEARCH_2_STEPS_PIVOT_FILTER]
                    threshold = 1
                else:
                    join_value = result.__getattr__(
                        config.SEARCH_2_STEPS_PIVOT_FILTER)
                    threshold = result.score

                if join_value and threshold > config.SEARCH_2_STEPS_STEP1_THRESHOLD:
                    params_steps_2.append((join_value, query_step_1))

            # Make results uniq
            params_steps_2 = set(params_steps_2)

            # Run steps 2 queries
            for join_value, query_step_1 in params_steps_2:
                # Set step 2 query filter from step 1 result
                filters_step_2 = filters.copy()
                filters_step_2[config.SEARCH_2_STEPS_PIVOT_FILTER] = join_value
                filters_step_2['type'] = config.SEARCH_2_STEPS_STEP2_TYPE
                results_step_2 = multiple_search(
                    [q + ' ' + query_step_1 for q in queries2],
                    limit=limit,
                    autocomplete=autocomplete,
                    **filters_step_2)
                append = False
                if results_step_2:
                    for result_step_2 in results_step_2:
                        if result_step_2.score > config.SEARCH_2_STEPS_STEP2_THRESHOLD:
                            append = True
                            ret.append(result_step_2)
                if not append:
                    # No usable result from steps 2, use steps 1 result
                    # Lower the score
                    result.score *= config.SEARCH_2_STEPS_STEP2_PENALITY_MULTIPLIER
                    if result.score > config.SEARCH_2_STEPS_STEP2_THRESHOLD:
                        ret.append(result)

        results_full = multiple_search([q + ' ' + query1 for q in queries2],
                                       limit=limit,
                                       autocomplete=autocomplete,
                                       **filters)

    for result in results_full:
        # Lower the score
        result.score *= config.SEARCH_2_STEPS_STEP2_PENALITY_MULTIPLIER

        ret.append(result)

    if ret:
        # Sort and limit results for all queries
        ret = sorted(ret, key=lambda k: k.score, reverse=True)[0:limit]
        # Make result uniq
        ids = []
        uniq = []
        for e in ret:
            if e.id not in ids:
                uniq.append(e)
                ids.append(e.id)
        return uniq
    else:
        return results1[0:limit]
Beispiel #55
0
def test_should_not_return_housenumber_if_number_is_also_in_name(housenumber):
    housenumber.update(name='rue du 11 Novembre')
    results = search('rue du 11 novembre')
    assert not results[0].housenumber
    results = search('11 rue du 11 novembre')
    assert results[0].housenumber == '11'
Beispiel #56
0
def test_should_keep_unchanged_name_as_default_label(factory):
    factory(name="Porte des Lilas")
    results = search("porte des lilas")
    str(results[0]) == "Porte des Lilas"
Beispiel #57
0
def test_should_do_autocomplete(street):
    street.update(name='rue de Wambrechies', city="Bondues")
    assert search('avenue wambre', autocomplete=True)
    assert search('wambre avenue', autocomplete=True)
Beispiel #58
0
    def on_get(self, req, resp, **kwargs):
        query = req.get_param('q')
        language = req.get_param('language') or 'zh'
        if not query:
            raise falcon.HTTPBadRequest('Missing query', 'Missing query')
        limit = req.get_param_as_int('limit') or 20  # use config
        autocomplete = req.get_param_as_bool('autocomplete')
        if autocomplete is None:
            # Default is True.
            # https://github.com/falconry/falcon/pull/493#discussion_r44376219
            autocomplete = True
        lon, lat = self.parse_lon_lat(req)
        center = None
        if lon and lat:
            center = (lon, lat)
        filters = self.match_filters(req)
        timer = time.perf_counter()
        try:
            results = search(query,
                             limit=limit,
                             autocomplete=autocomplete,
                             lat=lat,
                             lon=lon,
                             **filters)
        except EntityTooLarge as e:
            raise falcon.HTTPRequestEntityTooLarge(str(e))
        timer = int((time.perf_counter() - timer) * 1000)
        if not results:
            log_notfound(query)
        log_query(query, results)
        if config.SLOW_QUERIES and timer > config.SLOW_QUERIES:
            log_slow_query(query, results, timer)
        filtered_results = {}

        def sortbyindex(item):
            idx = item.name.lower().find(query.lower())
            if idx == -1:
                idx = 100
            return idx

        def sortbylang(item):
            if item.lang == language:
                return 0
            else:
                return 1

        def sortbylength(item):
            return len(item.name)

        results.sort(
            key=lambda x: (sortbylang(x), sortbyindex(x), sortbylength(x)))
        count = 0
        for r in results:
            if not filtered_results.get(r.name.lower()) and r.type[0] != 'R':
                filtered_results[r.name.lower()] = r
                count += 1
            elif not filtered_results.get(r.id) and r.type[0] == 'R':
                filtered_results[r.id] = r
                count += 1
            if not req.get_param_as_int('limit') and count == 5:
                break

        self.render(req,
                    resp,
                    list(filtered_results.values()),
                    query=query,
                    filters=filters,
                    center=center,
                    limit=limit)