def get_nearest_stops(key, lng, lat, radius=150, units='m', with_dist=False, with_coord=False, with_hash=False, sort=None): if not api_redis.exists('locations'): set_cache_stops() pieces = [key, lng, lat, radius, units] # XXX Can modify precision based on user radius. Default to 36 (~150m) ranges = geohash.expand_uint64(geohash.encode_uint64(float(lat), float(lng)), precision=36) # XXX This has no affect at this time if with_dist: pieces.append('WITHDIST') if with_coord: pieces.append('WITHCOORD') if with_hash: pieces.append('WITHHASH') if sort: pieces.append(sort) # TODO Some day this can be done with the Geo Redis commands pipe = api_redis.pipeline(transaction=False) for r in ranges: pipe.zrangebyscore(key, r[0], r[1]) # TODO This will require some intermediate steps if scores are returned stops = [stop for stop_list in pipe.execute() for stop in stop_list] # TODO Implement filtering by radius and units return get_stop_details(stops)
def geohash_values(self, lat, lon): hsh = geohash.encode(lat, lon) if lat and lon else None return { "hash": hsh, "hash_u64": geohash.encode_uint64(lat, lon) if lat and lon else None, "hash_bin": hsh[:self.bb_prefix] if hsh else None }
def build_filters(self, filters=None): if filters is None: filters = {} if 'range' in filters: ran = int(filters.pop('range')[0]) else: ran = 30 orm_filters = super(BeatResource, self).build_filters(filters) # QueryDict values are always list, get the first item lat = orm_filters.pop('lat__exact', (None,))[0] lng = orm_filters.pop('lng__exact', (None,))[0] if (lat is not None) and (lng is not None): gh = geohash.encode_uint64(float(lat), float(lng)) orm_filters.update({'geohash_exact': gh}) orm_filters.update({'range': ran}) return orm_filters
def __init__(self, filename, geohashSize): if filename: self.dictionary = dict() self.geodictionary = defaultdict(list) with open(filename, "rb") as csvfile: datareader = csv.reader(csvfile) next(datareader, None) # skip the headers for row in datareader: id = UUID(row[Column.id]) name = row[Column.name] lat = float(row[Column.lat]) long = float(row[Column.long]) geo_hash64 = geohash.encode_uint64(lat, long) geo_hash = geohash.encode(lat, long, 20) shop = Shop(id, lat, long, geo_hash) self.dictionary[id] = shop self.geodictionary[geo_hash[:geohashSize]].append(shop)
def test_one(self): dataset = [ (0x0000000000000000, -90.0, -180.0), (0x0800000000000000, -90.0, -135.0), (0x1000000000000000, -45.0, -180.0), (0x1800000000000000, -45.0, -135.0), (0x2000000000000000, -90.0, -90.0), (0x2800000000000000, -90.0, -45.0), (0x3000000000000000, -45.0, -90.0), (0x3800000000000000, -45.0, -45.0), (0x4000000000000000, 0.0, -180.0), (0x4800000000000000, 0.0, -135.0), (0x5000000000000000, 45.0, -180.0), (0x5800000000000000, 45.0, -135.0), (0x6000000000000000, 0.0, -90.0), (0x6800000000000000, 0.0, -45.0), (0x7000000000000000, 45.0, -90.0), (0x7800000000000000, 45.0, -45.0), (0x8000000000000000, -90.0, 0.0), (0x8800000000000000, -90.0, 45.0), (0x9000000000000000, -45.0, 0.0), (0x9800000000000000, -45.0, 45.0), (0xA000000000000000, -90.0, 90.0), (0xA800000000000000, -90.0, 135.0), (0xB000000000000000, -45.0, 90.0), (0xB800000000000000, -45.0, 135.0), (0xC000000000000000, 0.0, 0.0), (0xC800000000000000, 0.0, 45.0), (0xD000000000000000, 45.0, 0.0), (0xD800000000000000, 45.0, 45.0), (0xE000000000000000, 0.0, 90.0), (0xE800000000000000, 0.0, 135.0), (0xF000000000000000, 45.0, 90.0), (0xF800000000000000, 45.0, 135.0) ] for data in dataset: self.assertEqual(data[0], geohash.encode_uint64(data[1], data[2])) latlon = geohash.decode_uint64(data[0]) self.assertEqual(latlon[0], data[1]) self.assertEqual(latlon[1], data[2])
def index_restaurant_data(args): datafile = args.data host = args.host port = args.port pipe = redis.StrictRedis(host=host, port=port).pipeline() with open(datafile, 'r') as f: count = 0 for linum, line in enumerate(f): try: count += 1 rest = json.loads(line, 'ISO-8859-1') lat, lon = rest['location'] h = hash_location(lat, lon) key = 'geobox:%s:restaurant' % h pipe.zadd(key, geohash.encode_uint64(lat, lon), line.strip()) if count % 1000 == 0: pipe.execute() print '%d items has been inserted' % count except Exception as e: print str(linum) + ' ' + line raise e
def test_one(self): dataset = [(0x0000000000000000, -90.0, -180.0), (0x0800000000000000, -90.0, -135.0), (0x1000000000000000, -45.0, -180.0), (0x1800000000000000, -45.0, -135.0), (0x2000000000000000, -90.0, -90.0), (0x2800000000000000, -90.0, -45.0), (0x3000000000000000, -45.0, -90.0), (0x3800000000000000, -45.0, -45.0), (0x4000000000000000, 0.0, -180.0), (0x4800000000000000, 0.0, -135.0), (0x5000000000000000, 45.0, -180.0), (0x5800000000000000, 45.0, -135.0), (0x6000000000000000, 0.0, -90.0), (0x6800000000000000, 0.0, -45.0), (0x7000000000000000, 45.0, -90.0), (0x7800000000000000, 45.0, -45.0), (0x8000000000000000, -90.0, 0.0), (0x8800000000000000, -90.0, 45.0), (0x9000000000000000, -45.0, 0.0), (0x9800000000000000, -45.0, 45.0), (0xA000000000000000, -90.0, 90.0), (0xA800000000000000, -90.0, 135.0), (0xB000000000000000, -45.0, 90.0), (0xB800000000000000, -45.0, 135.0), (0xC000000000000000, 0.0, 0.0), (0xC800000000000000, 0.0, 45.0), (0xD000000000000000, 45.0, 0.0), (0xD800000000000000, 45.0, 45.0), (0xE000000000000000, 0.0, 90.0), (0xE800000000000000, 0.0, 135.0), (0xF000000000000000, 45.0, 90.0), (0xF800000000000000, 45.0, 135.0)] for data in dataset: self.assertEqual(data[0], geohash.encode_uint64(data[1], data[2])) latlon = geohash.decode_uint64(data[0]) self.assertEqual(latlon[0], data[1]) self.assertEqual(latlon[1], data[2])
def _encode(self, lat, lon): return geohash.encode_uint64(lat, lon)
def adaptive_geohash_nearby_search(latitude, longitude, lower_cut, upper_cut, select_cols, count_col, lat_col, lon_col, geoint_col, custom_where_clause, db_conn): ''' Given the latitude, longitude of a point of interest, the lower and upper cut in the result pagination, the columns to select, the primary key column, the latitude and longitude columns, the geohash column, a custom SQL WHERE clause string to add, and a SQLAlchemy database connection, returns a list of selected SQLAlchemy rows as requested in the range of lower_cut and upper_cut, ordered by distance to the point provided, ascending. Returns None if the geohash search failed (should fallback to other search methods). latitude: latitude of the point to search around. longitude: longitude of the point to search around. lower_cut: the starting slice index of the results wanted (same rule as list slicing). upper_cut: the ending slice index of the results wanted (same rule as list slicing). select_cols: list of SQLAlchemy table column objects to select in the result. count_col: a SQLAlchemy table column object that is in the table used to count potential results, the primary key column is recommended, but any other singly unique column would work. lat_col: a SQLAlchemy table column object, the latitude column in the table. lon_col: a SQLAlchemy table column object, the longitude column in the table. geoint_col: a SQLAlchemy table column object, the geohash column in the table. custom_where_clause: any custom SQL WHERE conditions to add in addition to the nearby query. Must be sanitized, and lead with the logic connector (e.g. AND) to join with the geohash queries. db_conn: a SQLAlchemy database connection, used to perform the database query. ''' if not (isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)) or not isinstance(lower_cut, int) or not isinstance(upper_cut, int)): raise InvalidParamException() if lat_col not in select_cols: select_cols.append(lat_col) if lon_col not in select_cols: select_cols.append(lon_col) max_result_cap = upper_cut # Geohash may give inaccurate results for smaller result numbers, so # make sure we at least get 20 rows, which can be discarded later max_result_cap = max(max_result_cap, 20) coord_geoint = geohash.encode_uint64(latitude, longitude) MAX_DB_HITS = 20 # A limit on the maximum number of repeated database queries before giving up precision = 50 # Starting precision previous_precision = 50 previous_result_size = 0 previous_where_clause = '' stop = False fall_back = False loop_count = 0 while not stop and not fall_back: where_clause = get_geohash_where_clause(coord_geoint, precision, geoint_col.name) if len(where_clause) == 0: # Cannot find anything, fallback to other search methods stop = True fall_back = True else: if len(custom_where_clause) > 0: where_clause += custom_where_clause data_count = db_conn.execute(select(columns=[func.count(count_col)], whereclause=where_clause)).first() result_size = data_count[0] loop_count += 1 if result_size < previous_result_size: # Precision getting too low, result size shrinking, stop and use the previous where clause where_clause = previous_where_clause stop = True else: previous_result_size = result_size previous_precision = precision previous_where_clause = where_clause if result_size >= max_result_cap: stop = True if result_size < max_result_cap and loop_count >= MAX_DB_HITS: # Hit the max database query limit, fall back to other search methods stop = True fall_back = True else: percent_returned = float(result_size) / float(max_result_cap) # Adaptively decrease the search precision based on the percent of rows # we have found, while making sure that the precision does not decrease # to too low if percent_returned == 0 and (1.0 / float(max_result_cap)) < 0.05: if precision <= 4: precision -= 1 else: precision -= 8 precision = max(4, precision) elif percent_returned == 0 and (1.0 / float(max_result_cap)) >= 0.05: if precision <= 4: precision -= 1 else: precision -= 7 precision = max(4, precision) elif percent_returned <= 0.05: if precision <= 4: precision -= 1 else: precision -= 6 precision = max(4, precision) elif percent_returned < 0.25: if precision <= 4: precision -= 1 else: precision -= 4 precision = max(4, precision) elif percent_returned < 0.5: if precision <= 3: precision -= 1 else: precision -= 2 precision = max(3, precision) else: precision -= 1 precision = max(0, precision) if not fall_back: # Actually fetch the results based on the last WHERE clause used, which is successful data = db_conn.execute(select(columns=select_cols, whereclause=where_clause)).fetchall() sorted_list = [] for row in data: sorted_list.append([row, distance(latitude, longitude, row[lat_col], row[lon_col])]) # Sort the results by distance sorted_list = sorted(sorted_list, key=lambda sort_elem : sort_elem[1]) return [i[0] for i in sorted_list][lower_cut:upper_cut] else: # Geohash search failed, signal the failure return None
def score(lng, lat, precision=BIT_COUNT): h = encode_uint64(lat, lng) >> (64-BIT_COUNT) return (h >> (BIT_COUNT-precision)) << (BIT_COUNT-precision)
def set_cache_stops(): stops = get_stops_on_routes() key = '{0}.{1}.{2}.{3}' pipe = api_redis.pipeline(transaction=False) for stop in stops: stop['lat'], stop['lng'] = get_parsed_coordinate(stop[LAT_KEY]), get_parsed_coordinate(stop[LNG_KEY]) pipe.set(key.format(stop['abbreviation'], stop['stopID'], stop['directionID'], stop['sequence']), yaml.safe_dump(stop)) # TODO Some day this can be done with the Geo Redis commands pipe.zadd('locations', key.format(stop['abbreviation'], stop['stopID'], stop['directionID'], stop['sequence']), geohash.encode_uint64(stop['lat'], stop['lng'])) pipe.execute()
def save(self, *args, **kwargs): if not self.geohash: self.geohash = geo_hash.encode_uint64(self.lat, self.lng) super(Place, self).save(*args, **kwargs)
for sensor_type in [("temperature", (-10, 40)), ("humidity", (30, 90))]: if random.random() < 0.5: city_num = random.randint(0, len(city) - 1) (device_name, latitude, longitude) = city[city_num] else: device_name = "Random" latitude = random.uniform(-80, 80) longitude = random.uniform(-180, 180) (sensor_name, (min_value, max_value)) = sensor_type message_data = {} if device_name == location: if sensor_type == "temperature": message_data["sensor_value"] = sense.get_temperature() else: message_data["sensor_value"] = sense.get_humidity() sense.show_message(location) else: message_data["sensor_value"] = min_value + ( max_value - min_value) * random.random() message_data["latitude"] = latitude message_data["longitude"] = longitude message_data["geohash"] = geohash.encode(latitude, longitude) message_data["geohash_uint64"] = geohash.encode_uint64( latitude, longitude) message_json = json.dumps(message_data) publish_topic = "myapp/" + device_name + "/sensor/" + sensor_name logger.debug(publish_topic + " -> " + message_json) myAWSIoTMQTTClient.publish(publish_topic, message_json, 1) time.sleep(0.2)