def check_place_contents(context, table, exact): """ Check contents of place/placex tables. Each row represents a table row and all data must match. Data not present in the expected table, may be arbitry. The rows are identified via the 'object' column which must have an identifier of the form '<NRW><osm id>[:<class>]'. When multiple rows match (for example because 'class' was left out and there are multiple entries for the given OSM object) then all must match. All expected rows are expected to be present with at least one database row. When 'exactly' is given, there must not be additional rows in the database. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: expected_content = set() for row in context.table: nid = NominatimID(row['object']) query = 'SELECT *, ST_AsText(geometry) as geomtxt, ST_GeometryType(geometry) as geometrytype' if table == 'placex': query += ' ,ST_X(centroid) as cx, ST_Y(centroid) as cy' query += " FROM %s WHERE {}" % (table, ) nid.query_osm_id(cur, query) assert cur.rowcount > 0, "No rows found for " + row['object'] for res in cur: if exact: expected_content.add((res['osm_type'], res['osm_id'], res['class'])) DBRow(nid, res, context).assert_row(row, ['object']) if exact: cur.execute('SELECT osm_type, osm_id, class from {}'.format(table)) assert expected_content == set([(r[0], r[1], r[2]) for r in cur])
def check_search_name_contents(context, exclude): """ Check contents of place/placex tables. Each row represents a table row and all data must match. Data not present in the expected table, may be arbitry. The rows are identified via the 'object' column which must have an identifier of the form '<NRW><osm id>[:<class>]'. All expected rows are expected to be present with at least one database row. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: for row in context.table: nid = NominatimID(row['object']) nid.row_by_place_id( cur, 'search_name', ['ST_X(centroid) as cx', 'ST_Y(centroid) as cy']) assert cur.rowcount > 0, "No rows found for " + row['object'] for res in cur: db_row = DBRow(nid, res, context) for name, value in zip(row.headings, row.cells): if name in ('name_vector', 'nameaddress_vector'): items = [x.strip() for x in value.split(',')] with context.db.cursor() as subcur: subcur.execute( """ SELECT word_id, word_token FROM word, (SELECT unnest(%s::TEXT[]) as term) t WHERE word_token = make_standard_name(t.term) and class is null and country_code is null and operator is null UNION SELECT word_id, word_token FROM word, (SELECT unnest(%s::TEXT[]) as term) t WHERE word_token = ' ' || make_standard_name(t.term) and class is null and country_code is null and operator is null """, (list( filter(lambda x: not x.startswith('#'), items)), list( filter(lambda x: x.startswith('#'), items)))) if not exclude: assert subcur.rowcount >= len(items), \ "No word entry found for {}. Entries found: {!s}".format(value, subcur.rowcount) for wid in subcur: present = wid[0] in res[name] if exclude: assert not present, "Found term for {}/{}: {}".format( row['object'], name, wid[1]) else: assert present, "Missing term for {}/{}: {}".fromat( row['object'], name, wid[1]) elif name != 'object': assert db_row.contains(name, value), db_row.assert_msg( name, value)
def check_place_addressline_exclude(context): """ Check that the place_addressline doesn't contain any entries for the given addressee/address item pairs. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: for row in context.table: pid = NominatimID(row['object']).get_place_id(cur) apid = NominatimID(row['address']).get_place_id(cur) cur.execute(""" SELECT * FROM place_addressline WHERE place_id = %s AND address_place_id = %s""", (pid, apid)) assert cur.rowcount == 0, \ "Row found for place %s and address %s" % (row['object'], row['address'])
def add_data_to_planet_relations(context): """ Add entries into the osm2pgsql relation middle table. This is needed for tests on data that looks up members. """ with context.db.cursor() as cur: for r in context.table: last_node = 0 last_way = 0 parts = [] if r['members']: members = [] for m in r['members'].split(','): mid = NominatimID(m) if mid.typ == 'N': parts.insert(last_node, int(mid.oid)) last_node += 1 last_way += 1 elif mid.typ == 'W': parts.insert(last_way, int(mid.oid)) last_way += 1 else: parts.append(int(mid.oid)) members.extend((mid.typ.lower() + mid.oid, mid.cls or '')) else: members = None tags = chain.from_iterable([(h[5:], r[h]) for h in r.headings if h.startswith("tags+")]) cur.execute("""INSERT INTO planet_osm_rels (id, way_off, rel_off, parts, members, tags) VALUES (%s, %s, %s, %s, %s, %s)""", (r['id'], last_node, last_way, parts, members, list(tags)))
def check_place_has_entry(context, table, oid): """ Ensure that no database row for the given object exists. The ID must be of the form '<NRW><osm id>[:<class>]'. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: NominatimID(oid).query_osm_id(cur, "SELECT * FROM %s where {}" % table) assert cur.rowcount == 0, \ "Found {} entries for ID {}".format(cur.rowcount, oid)
def check_search_name_has_entry(context, oid): """ Check that there is noentry in the search_name table for the given objects. IDs are in format '<NRW><osm id>[:<class>]'. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: NominatimID(oid).row_by_place_id(cur, 'search_name') assert cur.rowcount == 0, \ "Found {} entries for ID {}".format(cur.rowcount, oid)
def check_search_name_contents(context, exclude): """ Check contents of place/placex tables. Each row represents a table row and all data must match. Data not present in the expected table, may be arbitry. The rows are identified via the 'object' column which must have an identifier of the form '<NRW><osm id>[:<class>]'. All expected rows are expected to be present with at least one database row. """ tokenizer = tokenizer_factory.get_tokenizer_for_db( context.nominatim.get_test_config()) with tokenizer.name_analyzer() as analyzer: with context.db.cursor( cursor_factory=psycopg2.extras.DictCursor) as cur: for row in context.table: nid = NominatimID(row['object']) nid.row_by_place_id( cur, 'search_name', ['ST_X(centroid) as cx', 'ST_Y(centroid) as cy']) assert cur.rowcount > 0, "No rows found for " + row['object'] for res in cur: db_row = DBRow(nid, res, context) for name, value in zip(row.headings, row.cells): if name in ('name_vector', 'nameaddress_vector'): items = [x.strip() for x in value.split(',')] tokens = analyzer.get_word_token_info( context.db, items) if not exclude: assert len(tokens) >= len(items), \ "No word entry found for {}. Entries found: {!s}".format(value, len(tokens)) for word, token, wid in tokens: if exclude: assert wid not in res[name], \ "Found term for {}/{}: {}".format(nid, name, wid) else: assert wid in res[name], \ "Missing term for {}/{}: {}".format(nid, name, wid) elif name != 'object': assert db_row.contains(name, value), db_row.assert_msg( name, value)
def check_place_addressline(context): """ Check the contents of the place_addressline table. Each row represents a table row and all data must match. Data not present in the expected table, may be arbitry. The rows are identified via the 'object' column, representing the addressee and the 'address' column, representing the address item. """ with context.db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur: for row in context.table: nid = NominatimID(row['object']) pid = nid.get_place_id(cur) apid = NominatimID(row['address']).get_place_id(cur) cur.execute(""" SELECT * FROM place_addressline WHERE place_id = %s AND address_place_id = %s""", (pid, apid)) assert cur.rowcount > 0, \ "No rows found for place %s and address %s" % (row['object'], row['address']) for res in cur: DBRow(nid, res, context).assert_row(row, ('address', 'object'))
def delete_places(context, oids): """ Remove entries from the place table. Multiple ids may be given separated by commas. Also runs all triggers related to updates and reindexes the new data. """ context.nominatim.run_nominatim('refresh', '--functions') with context.db.cursor() as cur: for oid in oids.split(','): NominatimID(oid).query_osm_id(cur, 'DELETE FROM place WHERE {}') context.nominatim.reindex_placex(context.db)
def website_details_request(context, fmt, query): params = {} if query[0] in 'NWR': nid = NominatimID(query) params['osmtype'] = nid.typ params['osmid'] = nid.oid if nid.cls: params['class'] = nid.cls else: params['place_id'] = query outp, status = send_api_query('details', params, fmt, context) context.response = GenericResponse(outp, fmt or 'json', status)