def plot_map(df_all, title, pause=False):
df_all['lat'] = df_all.apply(lambda rec: geohash.decode(rec['geohash'])[1],
axis=1)
df_all['lon'] = df_all.apply(lambda rec: geohash.decode(rec['geohash'])[0],
axis=1)
gdf = gpd.GeoDataFrame(df_all,
geometry=gpd.points_from_xy(df_all.lat, df_all.lon))
world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
fig, ax = plt.subplots(1, 1, figsize=(15, 15))
fig.suptitle(
f'East African Hopper Locust detection via Sentenel 2 Imagery',
fontsize=20)
ax.set_title(title)
world[world.name.isin([
'Ethiopia', 'Kenya', 'Somalia', 'Eritrea', 'Yemen', 'Djibouti',
'Somaliland'
])].plot(color='white', edgecolor='Black', ax=ax)
gdf.loc[gdf['tn'] == 1].plot(ax=ax, color='blue')
gdf.loc[gdf['fp'] == 1].plot(ax=ax, color='red')
gdf.loc[gdf['fn'] == 1].plot(ax=ax, color='yellow')
gdf.loc[gdf['tp'] == 1].plot(ax=ax, color='green')
ax.legend(['tn', 'fp', 'fn', 'tp'], loc="upper right", title="legend")
if pause:
plt.show(block=False)
plt.pause(3)
plt.close()
def get_geohash_directions(gh_A,gh_B):
try:
with open("google_directions_cache.json","r") as f:
google_directions_cache = json.load(f)
except ValueError:
GPS_A = geohash.decode(gh_A)
GPS_B = geohash.decode(gh_B)
directions_result = gmaps.directions(GPS_A,
GPS_B,
mode="driving")
connection_data =({'distance':directions_result[0]['legs'][0]['distance']['value'],
'steps':len(directions_result[0]['legs'][0]['steps'])})
time.sleep(1)
print ("failed cache read")
return connection_data
sorted_hashes = sorted([gh_A,gh_B])
connection_key = sorted_hashes[0] + sorted_hashes[1]
if connection_key in list(google_directions_cache.keys()):
connection_data = google_directions_cache[connection_key]
else:
GPS_A = geohash.decode(gh_A)
GPS_B = geohash.decode(gh_B)
directions_result = gmaps.directions(GPS_A,
GPS_B,
mode="driving")
connection_data =({'distance':directions_result[0]['legs'][0]['distance']['value'],
'steps':len(directions_result[0]['legs'][0]['steps'])})
google_directions_cache[connection_key] = connection_data
with open("google_directions_cache.json","w") as f:
json.dump(google_directions_cache,f)
time.sleep(1)
return connection_data
def __init__(self, external_identifier, name, device_uuid=None, geohash=None, measures=None, tags=None,
parent_device_uuid=None, latitude=None, longitude=None):
""" initialize new device object
"""
# prevent inconsistant geospatial data from making it into repository
# rederive lat / long
if geohash and (latitude or longitude):
(latitude, longitude) = gh.decode(geohash)
# but populate both to facilitate queries that need lat/long data
if (latitude and longitude) and not geohash:
geohash = gh.encode(latitude, longitude)
if geohash and not (latitude or longitude):
(latitude, longitude) = gh.decode(geohash)
self.external_identifier = external_identifier
self.name = name
self.geohash = geohash
self.measures = measures
self.tags = tags
self.parent_device_id = parent_device_uuid
self.latitude = latitude
self.longitude = longitude
# get a uuid if one wasn't passed in.
if device_uuid is None:
self.device_uuid = uuid.uuid4()
elif isinstance(device_uuid, basestring):
self.device_uuid = uuid.UUID(device_uuid)
else:
self.device_uuid = device_uuid
def cal_distance(s,t):
#s,t = sorted([s,t])
#if (s,t) in cdic:
# return cdic[(s,t)]
if s in cdic:
lat1,lon1 = cdic[s]
else:
lat1,lon1 = decode(s)
cdic[s] = (lat1,lon1)
if t in cdic:
lat2,lon2 = cdic[t]
else:
lat2,lon2 = decode(t)
cdic[t] = (lat2,lon2)
#lat2,lon2 = decode(t)
dx = abs(lon1 - lon2)
dy = abs(lat1 - lat2)
b = (lat1 + lat2) / 2.0
Lx = 6371004.0 * (dx / 57.2958) * cos(b / 57.2958)
Ly = 6371004.0 * (dy / 57.2958)
L = (Lx**2 + Ly**2) ** 0.5
cdic[(s,t)] = L
return L
def distanceCal(code1, code2):
lat1, lon1 = geohash.decode(code1)
lat1 = math.radians(float(lat1))
lon1 = math.radians(float(lon1))
lat2, lon2 = geohash.decode(code2)
lat2 = math.radians(float(lat2))
lon2 = math.radians(float(lon2))
x = (lon2 - lon1) * math.cos((lat1 + lat2) / 2)
y = lat2 - lat1
distance = (math.sqrt(x * x + y * y)) * 6371
return distance
def plotGeoHashTree(mset,usePyLeaflet=False,save_to_file=None):
fig, ax = plt.subplots(figsize=(16, 10),nrows=1,ncols=1)
for s in mset:
lat1,lon1 = geohash.decode(s[2])
lat2,lon2 = geohash.decode(s[3])
ax.plot([lon1,lon2],[lat1,lat2],color='b',linewidth=3,alpha=.7)
if usePyLeaflet or (save_to_file is not None):
tile_layer = "http://{s}.tile.stamen.com/terrain/{z}/{x}/{y}.jpg"
pyLeaflet.plotWithMap(fig,ip="md-bdadev-149.verizon.com",tile_layer = tile_layer,saveAs=save_to_file)
else:
plt.show()
def import_mobike2db(mobike_csv_path):
with open(mobike_csv_path, 'r') as csvfile:
reader = csv.reader(csvfile)
rows = [row for row in reader]
rows = rows[1:]
sql_insert = "insert into `mobike` (`order_id`, `user_id`, `bike_id`, `bike_type`, `start_time`, `geohashed_start_loc`, `geohashed_end_loc`, `start_lat`, `start_lng`,`end_lat`,`end_lng`,`is_weekend`,`hour`) values"
count = 0
for row in rows:
count += 1
if count < 3200000:
continue
print(count)
if count >= 3300000:
break
order_id = row[0]
user_id = row[1]
bike_id = row[2]
bike_type = row[3]
start_time = row[4]
hashed_start_loc = row[5]
hashed_end_loc = row[6]
date = start_time.split(' ')[0]
hour = int(start_time.split(' ')[1].split(':')[0])
is_weekend = date_is_weekend(date)
start_ll = geohash.decode(hashed_start_loc)
start_lng = float(start_ll[1])
start_lat = float(start_ll[0])
end_ll = geohash.decode(hashed_end_loc)
end_lng = float(end_ll[1])
end_lat = float(end_ll[0])
sql_insert += '("%s","%s","%s","%s","%s","%s","%s",%f,%f,%f,%f,%d,%d),' % (
order_id, user_id, bike_id, bike_type, start_time,
hashed_start_loc, hashed_end_loc, start_lat, start_lng,
end_lat, end_lng, is_weekend, hour)
db = MySQLdb.connect(**get_db_config())
cursor = db.cursor()
try:
sql_insert = sql_insert[:-1]
with open('sql.txt', 'w') as f:
f.write(sql_insert)
# print(str(sql_insert))
cursor.execute(sql_insert)
db.commit()
print("success")
except Exception as e:
print(e)
db.rollback()
def get_close_ghs(src_hash, lookup_hash_list, gh_precision, exp_iterations,
max_haversine):
exp_src_hash = gh_expansion(src_hash[0:gh_precision], exp_iterations)
if max_haversine == -1:
return [
gh for gh in lookup_hash_list if gh[0:gh_precision] in exp_src_hash
]
else:
return [
gh for gh in lookup_hash_list if gh[0:gh_precision] in exp_src_hash
and haversine(*reverse_GPS(geohash.decode(src_hash)),
*reverse_GPS(geohash.decode(gh))) <= max_haversine
]
def getDistance(geoHash1, geoHash2):
"""
Estimate the distance between 2 geohashes in uint64 format
"""
# return abs(geoHash1 - geoHash2)
try:
coords1 = hasher.decode(geoHash1)
coords2 = hasher.decode(geoHash2)
return math.sqrt(math.pow(coords1[0] - coords2[0], 2) +
math.pow(coords1[1] - coords2[1], 2))
except Exception, e:
print e
return None
def help_bike_leak(group):
start_loc = list(group['geohashed_start_loc'])
end_loc = list(group['geohashed_end_loc'])
time = list(group['starttime'])
next_time_delta = []
next_s_s_dis = []
# next_e_e_dis = []
next_s_e_dis = []
next_e_s_dis = []
# bike_s_s_v = []
# bike_e_e_v = []
for i in range(len(start_loc) - 1):
lat_start_i, lon_start_i = geohash.decode(start_loc[i])
lat_start_i2, lon_start_i2 = geohash.decode(start_loc[i + 1])
lat_end_i, lon_end_i = geohash.decode(end_loc[i])
lat_end_i2, lon_end_i2 = geohash.decode(end_loc[i])
## 本条起点与下条起点的距离差
next_s_s_dis.append(
cal_distance(lat_start_i, lon_start_i, lat_start_i2, lon_start_i2))
## 本条终点与下条终点的距离差
# next_e_e_dis.append(cal_distance(lat_end_i, lon_end_i,lat_end_i2, lon_end_i2))
## 本条起点与下条终点的距离差
next_s_e_dis.append(
cal_distance(lat_start_i, lon_start_i, lat_end_i2, lon_end_i2))
## 本条终点与下条起点的距离差
next_e_s_dis.append(
cal_distance(lat_end_i, lon_end_i, lat_start_i2, lon_start_i2))
## 本条记录与下条记录的时间差
# next_time_delta.append((datetime.strptime(time[i+1], "%Y-%m-%d %H:%M:%S")-datetime.strptime(time[i], "%Y-%m-%d %H:%M:%S")).total_seconds())
next_time_delta.append((time[i + 1] - time[i]).total_seconds())
# bike_s_s_v = [i / j for i, j in zip(next_s_s_dis, next_time_delta)]
# bike_e_e_v = [i / j for i, j in zip(next_e_e_dis, next_time_delta)]
next_s_s_dis.append(0)
# next_e_e_dis.append(0)
next_s_e_dis.append(0)
next_e_s_dis.append(0)
next_time_delta.append(1000000)
# bike_s_s_v.append(0)
# bike_e_e_v.append(0)
group['next_s_s_dis'] = next_s_s_dis
# group['next_e_e_dis'] = next_e_e_dis
group['next_s_e_dis'] = next_s_e_dis
group['next_e_s_dis'] = next_e_s_dis
group['next_time_delta'] = next_time_delta
# group['bike_s_s_v'] = bike_s_s_v
# group['bike_e_e_v'] = bike_e_e_v
return group
def extract_location(train_op):
train_op['longitude'] = train_op['geo_code'].apply(
lambda x: geohash.decode(x)[0] if isinstance(x, str) else np.nan)
train_op['latitude'] = train_op['geo_code'].apply(
lambda x: geohash.decode(x)[1] if isinstance(x, str) else np.nan)
train_op[['longitude', 'latitude']] = train_op[['longitude',
'latitude']].astype(float)
train_op['longitude_diff'] = train_op.groupby(['UID'])['longitude'].diff()
train_op['latitude_diff'] = train_op.groupby(['UID'])['latitude'].diff()
train_op['longitude_day_diff'] = train_op.groupby(['UID', 'day'
])['longitude'].diff()
train_op['latitude_day_diff'] = train_op.groupby(['UID', 'day'
])['latitude'].diff()
return train_op
def encdec():
print 'geohash.encode ' + geohash.encode(12.963787, 77.637789)
print 'geohash.encode with precision-5: ' + geohash.encode(
12.963787, 77.637789, precision=5)
print 'geohash.decode ' + str(geohash.decode('tdr1wxype953'))
print 'geohash.decode exactly' + str(
geohash.decode_exactly('tdr1wxype953'))
def get_cells_in_circle(lat, lon, radius, precision):
"""Gets all geohash cells inside a circle, sorted approximately by distance.
Args:
lat: float, the latitude of the circle center.
lon: float, the longitude of the circle center.
radius: float, the radius of the circle in meters.
precision: int, the precision of the geohash.
Returns:
list, the list of geohash cells.
"""
# Get all cells that are in the circle (with the max_resolution).
# Start from the center cell.
cur_set = set([geohash.encode(lat, lon, precision)])
all_set = set(cur_set)
result = list(cur_set)
while cur_set:
# Gradually extend the found cells (all_set) layer by layer.
new_set = set([])
for cell in cur_set:
for one_neighbor in geohash.neighbors(cell):
if one_neighbor in all_set:
continue
(nb_lat, nb_lon) = geohash.decode(one_neighbor)
if distance(nb_lat, nb_lon, lat, lon) < radius:
new_set.add(one_neighbor)
all_set.update(new_set)
result.extend(list(new_set))
cur_set = new_set
return result
def decode_geohash(geo):
""" Decode geohash into Lat/Lng coordinates """
coordinates = geohash.decode( \
geo \
)
return coordinates
def load_and_process_training_set(file_path):
'''
load training set and apply some preprocessing
:param file_path:
:return:
'''
print('Loading and transforming dataset')
scaler = MinMaxScaler()
training_set = pd.read_csv(file_path)
training_set['lat_lon'] = training_set['geohash6'].apply(lambda x: geohash.decode(x))
training_set['lat'] = training_set['lat_lon'].apply(lambda x: x[0])
training_set['lon'] = training_set['lat_lon'].apply(lambda x: x[1])
training_set['hour'] = training_set['timestamp'].apply(lambda x: x.split(':')[0]).astype('int')
training_set['timestamp_hour'] = training_set.apply(get_timestamp_hour, axis=1)
training_set['lat_scaled'] = scaler.fit_transform(training_set['lat'].values.reshape(-1, 1))
training_set['lon_scaled'] = scaler.fit_transform(training_set['lon'].values.reshape(-1, 1))
training_set['d_t'] = training_set['demand']
training_set['sample_weight'] = training_set['demand'].apply(get_exp_sample_weight)
training_set['timestamp_decimal'] = training_set['timestamp'].apply(get_timestamp_decimal)
training_set['timestamp_decimal_scaled'] = scaler.fit_transform(
training_set['timestamp_decimal'].values.reshape(-1, 1))
training_set = training_set.sort_values(by=['geohash6', 'timestamp_hour']).reset_index(drop=True)
training_set = training_set.drop(columns='lat_lon')
print('Loading and transformation complete')
return training_set
def add_connections(self,src_hash):
connections = {}
node_hashes = geo_tools.get_close_ghs(src_hash,self.nodes(),3,2,MAX_RANGE)
for connected_node in node_hashes:
if connected_node != src_hash:
try:
self.add_edge(src_hash,connected_node,{key:data for key,data in self.connected_network[src_hash][connected_node].items()})
self[src_hash][connected_node]['first_node'] = str(min(int(self.node[src_hash]["SC_index"]), #order of connection
int(self.node[connected_node]["SC_index"])))
self[src_hash][connected_node]['second_node'] = str(max(int(self.node[src_hash]["SC_index"]), #order of connection
int(self.node[connected_node]["SC_index"])))
except KeyError as e:
src_GPS = geo_tools.reverse_GPS(geohash.decode(src_hash))
connection = {}
connection['node'] = connected_node
connection['directions'] = geo_tools.get_geohash_directions(src_hash,connected_node)
if connection['directions']['distance']/1000 <= MAX_RANGE:
edge_weight = self.get_edge_weight(src_hash,connection['node'])
self.add_edge(src_hash,connection['node'],{'weight':edge_weight,'distance':connection['directions']['distance'],
'steps':connection['directions']['steps'],
'first_node':str(min(int(self.node[src_hash]["SC_index"]), #order of connection
int(self.node[connection['node']]["SC_index"]))),
'second_node':str(max(int(self.node[src_hash]["SC_index"]), #order of connection
int(self.node[connection['node']]["SC_index"]))),
'lon_lat_1':geo_tools.reverse_GPS(geohash.decode(src_hash)),
'lon_lat_2':geo_tools.reverse_GPS(geohash.decode(connection['node']))})
self.connected_network.add_edge(src_hash,connection['node'],{key:data for key,data in self[src_hash][connection['node']].items()})
return self
def decode(self, value):
"""
decode geohash value to latitude, longitude
:param value: geohash value to be decoded
:return: latitude, longitude pair for the center of the given geohash
"""
return geohash.decode(value)
def geohash_to_grid_point(self, geohash_list_file, output_file):
hash_file = geohash_list_file
with open(hash_file) as f:
content = f.readlines()
geo_dict = {}
for i in range(0, len(content)):
cur_code = content[i][:6]
geo_dict.update({cur_code: geohash.decode(cur_code)})
latitude = set()
longitude = set()
for key in geo_dict:
latitude.add(geo_dict[key][0])
longitude.add(geo_dict[key][1])
latitude = list(latitude)
latitude.sort()
longitude = list(longitude)
longitude.sort()
output = open(output_file, 'w')
for key in geo_dict:
lat = geo_dict[key][0]
lng = geo_dict[key][1]
output.write(key)
output.write('\t')
output.write(str(latitude.index(lat)))
output.write('\t')
output.write(str(longitude.index(lng)))
output.write('\n')
return 0
def test_cycle(self):
for code in [
"000000000000",
"zzzzzzzzzzzz",
"bgr96qxvpd46",
]:
self.assertEqual(code, geohash.encode(*geohash.decode(code)))
def filterFromSignature(data,hmap,maxDistance=5,heatThreshold=.01):
ndata = copy.deepcopy(data)
geohashlength = hmap['geohashlength']
for l in range(len(data)):
h = geohash.encode(data[l]['lat'],data[l]['lon'],geohashlength)
visited = set([h])
neighbors = set([h])
nonZeroNeighbors = [h] if (h in hmap and hmap[h]>heatThreshold) else []
d=0
while (len(nonZeroNeighbors)==0 and d<maxDistance):
nneighbors = set([])
for n in neighbors:
nneighbors.union([h for h in geohash.neighbors(n) if h not in visited])
neighbors = nneighbors
for n in neighbors:
if (n in hmap and hmap[n]>heatThreshold):
nonZeroNeighbors.append(n)
visited.union(neighbors)
d+=1
if len(nonZeroNeighbors)>0:
if len(nonZeroNeighbors)>1:
print h,nonZeroNeighbors
lat,lon=0.,0.
for n in nonZeroNeighbors:
dlat,dlon = geohash.decode(n)
lat += dlat
lon += dlon
ndata[l]= Location(lat/len(nonZeroNeighbors),
lon/len(nonZeroNeighbors),
data[l].timestamp)
return ndata
def get(ip, redisConn):
"""
Get a range and all its data by ip
"""
ipnum = IPRange.ip2long(ip)
#get the location record from redis
record = redisConn.zrangebyscore(IPRange._indexKey, ipnum, '+inf', 0, 1, True)
if not record:
#not found? k!
return None
#extract location id
try:
geoKey, rng = record[0][0].split('@')
lat, lon = hasher.decode(geoKey)
rngMin, rngMax, zipcode, country = rng.split(':')
rngMin = int(rngMin)
rngMax = int(rngMax)
except IndexError:
return None
#address not in any range
if not rngMin <= ipnum <= rngMax:
return None
return IPRange(rngMin, rngMax, lat, lon, zipcode, country)
def geo_hash_to_geo_point(hash_int):
if not isinstance(hash_int, numbers.Number):
raise GeoHashConversionException("`hash_int` should be a number")
if hash_int >= 0:
raise GeoHashConversionException("`hash_int` should be negative")
hashcode = hex(int(hash_int + 2**63))[2:].decode("hex")
return geohash.decode(hashcode)
def getN(geo) -> list:
w, j = geohash.decode(geo)
wdelta = [0.0027, -0.0027, 0]
jdelta = [0.0055, -0.0055, 0]
return [
geohash.encode(x * 2 + w, y * 2 + j) for x in wdelta for y in jdelta if x != y
]
def get_3kilo_neighor(place):
pnum = geohash.decode(place)
res = list()
res.append(geohash.encode(pnum[0], pnum[1] - 0.01))
res.append(geohash.encode(pnum[0] - 0.02, pnum[1]))
res.append(geohash.encode(pnum[0] + 0.02, pnum[1]))
res.append(geohash.encode(pnum[0], pnum[1] + 0.01))
return res
def test_one_row_2db(mobike_csv_path):
with open(mobike_csv_path, 'r') as csvfile:
reader = csv.reader(csvfile)
rows = [row for row in reader]
rows = rows[1:]
row = rows[0]
print(row)
order_id = row[0]
user_id = row[1]
bike_id = row[2]
bike_type = row[3]
start_time = row[4]
hashed_start_loc = row[5]
hashed_end_loc = row[6]
date = start_time.split(' ')[0]
hour = int(start_time.split(' ')[1].split(':')[0])
is_weekend = date_is_weekend(date)
start_ll = geohash.decode(hashed_start_loc)
start_lng = float(start_ll[1])
start_lat = float(start_ll[0])
end_ll = geohash.decode(hashed_end_loc)
end_lng = float(end_ll[1])
end_lat = float(end_ll[0])
sql_insert = "replace into `mobike` (`order_id`, `user_id`, `bike_id`, `bike_type`, `start_time`, `geohashed_start_loc`, `geohashed_end_loc`, `start_lat`, `start_lng`,`end_lat`,`end_lng`,`is_weekend`,`hour`) values"
sql_insert += '("%s","%s","%s","%s","%s","%s","%s",%f,%f,%f,%f,%d,%d),' % (
order_id, user_id, bike_id, bike_type, start_time,
hashed_start_loc, hashed_end_loc, start_lat, start_lng, end_lat,
end_lng, is_weekend, hour)
print(sql_insert)
db = MySQLdb.connect(**get_db_config())
cursor = db.cursor()
try:
sql_insert = sql_insert[:-1]
# with open('1.txt', 'w') as f:
# f.write(sql_insert)
# print(str(sql_insert))
cursor.execute(sql_insert)
db.commit()
except Exception as e:
print(e)
db.rollback()
def getMarks(gHash, userUuid):
coords = geohash.decode(gHash)
x = coords[0] * 1112.0
y = coords[1] * 1112.0
gHashes = hashesToSearch(x, y)
dtThreshold = datetime.utcnow() - timedelta(hours=6)
data = [mark['value'] for mark in mongoMarks.find({'geoHash': {'$in': list(gHashes)}, 'utcDate': {'$gt': dtThreshold}})]
return json.dumps(data)
def get_close_ghs(src_hash,lookup_hash_list,gh_precision,exp_iterations,max_haversine):
exp_src_hash = gh_expansion(src_hash[0:gh_precision],exp_iterations)
if max_haversine == -1:
return [gh for gh in lookup_hash_list if gh[0:gh_precision] in exp_src_hash]
else:
return [gh for gh in lookup_hash_list
if gh[0:gh_precision] in exp_src_hash
and haversine(*reverse_GPS(geohash.decode(src_hash)),*reverse_GPS(geohash.decode(gh))) <= max_haversine]
def proximity_search(self, latitude, longitude, radius):
"""
Given a centerpoint, find everything within a radius around
that latitude and longitude, returned in order.
:param latitude: floating point latitude
:param longitude: floating point longitude
:param radius: radius in meters.
:return:
"""
hashcode = geohash.encode(latitude=latitude, longitude=longitude)
centerpoint = (latitude, longitude)
tmp_hashcode = ''
for x in hashcode:
# Go through the hashcode character by character
tmp_hashcode += x
lat, lng, delta_lat, delta_lng = geohash.decode(tmp_hashcode,
delta=True)
overall_lat = 2 * 1000 * haversine(
point1=(latitude - delta_lat, longitude),
point2=(latitude + delta_lat, longitude)
)
overall_lng = 2 * 1000 * haversine(
point1=(latitude, longitude-delta_lng),
point2=(latitude, longitude+delta_lng)
)
dist = min(overall_lng, overall_lat)
if dist < radius:
tmp_hashcode = tmp_hashcode[:-1]
break
if tmp_hashcode == '':
raise ValueError('Radius larger than earth')
precision = len(tmp_hashcode)
search_hashes = self._get_adjoining_hashes(hashcode=hashcode,
precision=precision)
search_hashes.append(tmp_hashcode)
possible_points = []
result_values = []
for search_hash in search_hashes:
possible_points.extend(self.storage.values(prefix=search_hash))
for point_id in possible_points:
point = self.points_by_id[point_id]
dist = 1000 * haversine(centerpoint, point)
if dist <= radius:
result_values.append((point_id, dist))
sorted_results = sorted(result_values, key = lambda x: x[1])
final_results = [x[0] for x in sorted_results]
return final_results
def proximity_search(self, latitude, longitude, radius):
"""
Given a centerpoint, find everything within a radius around
that latitude and longitude, returned in order.
:param latitude: floating point latitude
:param longitude: floating point longitude
:param radius: radius in meters.
:return:
"""
hashcode = geohash.encode(latitude=latitude, longitude=longitude)
centerpoint = (latitude, longitude)
tmp_hashcode = ''
for x in hashcode:
# Go through the hashcode character by character
tmp_hashcode += x
lat, lng, delta_lat, delta_lng = geohash.decode(tmp_hashcode,
delta=True)
overall_lat = 2 * 1000 * haversine(
point1=(latitude - delta_lat, longitude),
point2=(latitude + delta_lat, longitude))
overall_lng = 2 * 1000 * haversine(
point1=(latitude, longitude - delta_lng),
point2=(latitude, longitude + delta_lng))
dist = min(overall_lng, overall_lat)
if dist < radius:
tmp_hashcode = tmp_hashcode[:-1]
break
if tmp_hashcode == '':
raise ValueError('Radius larger than earth')
precision = len(tmp_hashcode)
search_hashes = self._get_adjoining_hashes(hashcode=hashcode,
precision=precision)
search_hashes.append(tmp_hashcode)
possible_points = []
result_values = []
for search_hash in search_hashes:
possible_points.extend(self.storage.values(prefix=search_hash))
for point_id in possible_points:
point = self.points_by_id[point_id]
dist = 1000 * haversine(centerpoint, point)
if dist <= radius:
result_values.append((point_id, dist))
sorted_results = sorted(result_values, key=lambda x: x[1])
final_results = [x[0] for x in sorted_results]
return final_results
def third_one():
name = request.form['name']
street = request.form['street']
city = request.form['city']
state = request.form['state']
country = request.form['country']
street = street.replace(" ", "_")
city = city.replace(" ", "_")
state = state.replace(" ", "_")
country = country.replace(" ", "_")
if name == "" or name == None:
name = "random ass person"
location_total = street + "," + city + "," + state + "," + country
google_url = "http://maps.googleapis.com/maps/api/geocode/json?address=" + location_total + "&sensor=false"
google_data = json.loads(urllib2.urlopen(google_url).read())
if google_data["status"] != "OK":
return render_template('ErrorMessage.html', name = name)
lat = google_data["results"][0]["geometry"]["location"]["lat"]
lon = google_data["results"][0]["geometry"]["location"]["lng"]
link = "http://runextbus.herokuapp.com/locations"
data_parsed = json.loads(urllib2.urlopen(link).read())
keys = []
for i in data_parsed.keys():
keys.append(i)
latitude = []
longitude = []
bus_name = []
for j in range(0, len(keys)):
for i in range(0, len(data_parsed[keys[j]])):
latitude.append(float(data_parsed[keys[j]][i]["lat"]))
longitude.append(float(data_parsed[keys[j]][i]["lon"]))
#return str(data_parsed[keys[j]][i]["lat"])
#return str(data_parsed[keys[j]][i]["lon"])
new = str(data_parsed[keys[j]][i]["dirtag"])
new = re.compile("[^\w']|_").sub(" ", new)
bus_name.append(new)
#return str(len(latitude))
#return jsonify(data_parsed)
data_link = "http://runextbus.herokuapp.com/active"
data_link = json.loads(urllib2.urlopen(data_link).read())
#return str(data_link["stops"][0]["geoHash"])
#return str(data_link["stops"][0]["title"])
#return str(len(data_link["stops"]))
#return jsonify(data_link)
geolat = []
geolon = []
bus_stop = []
for i in range(0, len(data_link["stops"])):
post = data_link["stops"][i]["geoHash"]
post = str(post)
temp = geohash.decode(post)
geolat.append(float(temp[0]))
geolon.append(float(temp[1]))
hold = data_link["stops"][i]["title"]
bus_stop.append(str(hold))
return render_template('mapmarkers.html', name = name, lat = lat, lon = lon, latitude = latitude, longitude = longitude, bus_name = json.dumps(bus_name), geolat = geolat, geolon = geolon, bus_stop = json.dumps(bus_stop))
def lookup():
if 'postcode' not in request.args:
return jsonify(detail="Please supply a 'postcode' parameter", about="http://github.com/j4mie/postcodeserver"), 400
try:
postcode = request.args['postcode'].replace(' ', '').upper()
hash = db[postcode][0].decode()
lat, lng = geohash.decode(hash)
return jsonify(postcode=postcode, geohash=hash, lat=lat, lng=lng)
except:
return jsonify(detail="Failed to lookup postcode"), 400
def lookup():
if 'postcode' not in request.args:
return jsonify(detail="Please supply a 'postcode' parameter", about="http://github.com/j4mie/postcodeserver"), 400
try:
postcode = request.args['postcode'].replace(' ', '').upper()
hash = db[postcode][0]
lat, lng = geohash.decode(hash)
return jsonify(postcode=postcode, geohash=hash, lat=lat, lng=lng)
except:
return jsonify(detail="Failed to lookup postcode"), 400
def geohash_bbox_query(extent, geohash_list, precision):
tl_hash = geohash.encode(extent[3], extent[0], precision=precision)
tr_hash = geohash.encode(extent[3], extent[1], precision=precision)
br_hash = geohash.encode(extent[2], extent[1], precision=precision)
bl_hash = geohash.encode(extent[2], extent[0], precision=precision)
com = common_hash(tl_hash, tr_hash, br_hash, bl_hash)
intersecting_hashes = [x for x in geohash_list if x.startswith(com)]
centroids = [geohash.decode(x)
for x in intersecting_hashes] #Note these are (y,x)
return filter_remaining(intersecting_hashes, centroids, extent)
def cluster_feature(train, result):
result['e_lon'] = result['geohashed_end_loc'].apply(
lambda x: geohash.decode(x)[1])
result['e_lat'] = result['geohashed_end_loc'].apply(
lambda x: geohash.decode(x)[0])
coords = np.vstack(
(train[['s_lon', 's_lat']].values, train[['e_lon', 'e_lat']].values))
# sample_ind = np.random.permutation(len(coords))[:800000]
# kmeans = MiniBatchKMeans(n_clusters=50, batch_size=10000).fit(coords[sample_ind])
kmeans = MiniBatchKMeans(n_clusters=20,
batch_size=10000).fit(coords[:800000])
# kmeans = KMeans(n_clusters=40).fit(coords[:800000])
result.loc[:, 's_cluster'] = kmeans.predict(result[['s_lon', 's_lat']])
result.loc[:, 'e_cluster'] = kmeans.predict(result[['e_lon', 'e_lat']])
# s_cluster = pd.get_dummies(result['s_cluster'], prefix='s', prefix_sep='_')
# e_cluster = pd.get_dummies(result['e_cluster'], prefix='e', prefix_sep='_')
# result = pd.concat([result,s_cluster],axis=1)
# result = pd.concat([result,e_cluster],axis=1)
return result
def limit_coordinates(coordinates, precision=5):
'''
Combines coordinates based on geohash precision
:param coordinates: list of (lat, lon) tuples
:param precision: geohash precision
:return:
'''
hashes = [(lat, lon, encode(lat, lon, precision)) for lat, lon in coordinates]
hashdict = dict([(h, decode(h)) for h in set([h[2] for h in hashes])])
return hashes, hashdict
def start_analysis_mission(db_name, limition=False):
print('开始分析数据:', db_name)
lon = None
lat = None
if limition is True:
lat, lon = geohash.decode(db_name)
analyzer = topline.Analyzer(db_name, lon, lat, 3)
analyzer.generate()
def start_analysis_mission(db_name, limition=False):
print('开始分析数据:', db_name)
lon = None
lat = None
if limition is True:
lat,lon = geohash.decode(db_name)
analyzer = topline.Analyzer(db_name, lon, lat, 3)
analyzer.generate()
def __str__(self):
latlng = geohash.decode(self.geo_hash)
ret = {
"geo_hash": self.geo_hash,
"demand": self.demand,
"supply": self.supply,
"surge": self.surge,
"lat": latlng[0],
"lng": latlng[1]
}
return json.dumps(ret)
def closestDoctorByDistance(nearby_doctors_list): distance_result_list = [] lat = '' long = '' for i in nearby_doctors_list: (lat, long) = geohash.decode(i[1]) distance_result = distance.distance(lat, long).meters print distance_result distance_result_list.append(distance_result) closest_doctor_by_distance = min(distance_result_list) return closestDoctorByDistance
def execute(self, current_location: Location) -> Optional[Location]:
"""Execution of the Movement Evaluation Policy"""
if random.random() <= settings.COURIER_MOVEMENT_PROBABILITY:
current_geohash = geohash.encode(*current_location.coordinates, precision=6)
geohash_neighbors = geohash.neighbors(current_geohash)
destination_geohash = random.choice(geohash_neighbors)
destination_coordinates = geohash.decode(destination_geohash)
return Location(lat=destination_coordinates[0], lng=destination_coordinates[1])
return None
def add_latlngs(data): newlist = [] lats =[] longs = [] for row in data['GEOHASH']: lat,long = geohash.decode(row) longs.append(long) lats.append(lat) data['LAT'] = lats data['LONG'] = longs return data
def _build_cell_tiles_from_shapefile(self, shpfile):
'''Computes all geohash tiles in the given shapefile
:param shapefile: shapefile
:return: a list of geohashes
'''
''' open shapefile'''
sf = shapefile.Reader(shpfile)
'''get features'''
shapes = sf.shapes()
if len(shapes) > 1:
print("More than one feature was found. Only first will be selected.")
input("Press Enter to continue...")
'''only use first feature'''
first_shp = shapes[0]
''' get shape type. only if shapetype is polygon'''
shape_type = first_shp.shapeType
if shape_type != 5:
handle_error(msg='Shapefile feature be a polygon')
''' get points coordinates for each point in the shape '''
points = first_shp.points
polygon = Polygon(points)
checked_geohashes = set()
geohash_stack = set()
geohashes = []
'''get center of bounding box, assuming the earth is flat'''
center_latitude = polygon.centroid.coords[0][1]
center_longitude = polygon.centroid.coords[0][0]
center_geohash = self.build_cell([center_longitude, center_latitude])
geohashes.append(center_geohash)
geohash_stack.add(center_geohash)
checked_geohashes.add(center_geohash)
while len(geohash_stack) > 0:
current_geohash = geohash_stack.pop()
neighbors = geohash.neighbors(current_geohash)
for neighbor in neighbors:
point = Point(geohash.decode(neighbor)[::-1])
if neighbor not in checked_geohashes and polygon.contains(point):
geohashes.append(neighbor)
geohash_stack.add(neighbor)
checked_geohashes.add(neighbor)
geohashes.sort()
return geohashes
def get_dist_df(geo_df):
geo_df.loc[:, 'lat'] = geo_df['geohash6'].apply(
lambda x: geohash.decode(x)[0])
geo_df.loc[:, 'lng'] = geo_df['geohash6'].apply(
lambda x: geohash.decode(x)[1])
geo_df['cost'] = 0.0
geo2_df = pd.merge(geo_df, geo_df, on='cost', how='outer')
geo2_df.loc[:,'cost'] = geo2_df[['lat_x', 'lng_x', 'lat_y', 'lng_y']].\
apply(lambda x: great_circle((x[0], x[1]), (x[2], x[3])).km, axis=1)
dist_df = geo2_df.loc[:, ['geohash6_x', 'geohash6_y', 'cost']].rename(
columns={
'geohash6_x': 'from',
'geohash6_y': 'to'
})
return dist_df
def get_geo2ll(geo_address):
"""
geohash 转经纬度，根据https://github.com/vinsci/geohash，该经纬度坐标体系应为WGS84
:param geo_address: geohash
:return: WGS84坐标系下的经纬度
"""
try:
lat, lng = geohash.decode(geo_address)
lat, lng = float(lat), float(lng)
except:
lat, lng = -1, -1
return [lng, lat]
def hmapToGraph(hmap,heatThreshold=-float('Inf')):
validKeys = [h for h in hmap.keys() if hmap[h]>heatThreshold and h not in ['minlon','minlat','maxlon','maxlat','maxvalue','geohashlength']]
points = np.array([geohash.decode(h) for h in validKeys])
tri = Delaunay(points)
# neighbors = dict(zip(validKeys, [[] for x in validKeys]))
# print neighbors
graph = {'vertices':validKeys,
'edges':set([])}
for t in tri.simplices:
for i in range(3):
graph['edges'].add((geohash_exactDistance(validKeys[t[i]],validKeys[t[(i+1)%3]]),hmap[validKeys[t[i]]]+hmap[validKeys[t[(i+1)%3]]],validKeys[t[i]],validKeys[t[(i+1)%3]]))
return graph
def _build_parameters(self):
parameters = {
'method': 'geo.getevents',
'limit': 25,
'api_key': API_KEY,
'format': 'json'}
if self.geohash and GEOHASH:
lon, lat = geohash.decode(self.geohash)
parameters['long'] = lon
parameters['lat'] = lat
if self.location:
parameters['location'] = self.location
return parameters
def to_python(self, value):
if value is None:
return None
if isinstance(value, (list, tuple)):
value = list(reversed(value))
if isinstance(value, string_types):
if self.LAT_LON_SEPARATOR in value:
value = list(value.split(self.LAT_LON_SEPARATOR))
elif GEOHASH_IMPORTED:
value = list(geohash.decode(value))
elif isinstance(value, dict):
value = [value.get('lat'), value.get('lon')]
return {'lat': float(value[0]), 'lon': float(value[1])}
def to_python(self, value):
if value is None:
return None
if isinstance(value, (list, tuple)):
value = list(reversed(value))
if isinstance(value, string_types):
if self.LAT_LON_SEPARATOR in value:
value = list(value.split(self.LAT_LON_SEPARATOR))
elif GEOHASH_IMPORTED:
value = list(geohash.decode(value))
elif isinstance(value, dict):
value = [value.get('lat'), value.get('lon')]
return {'lat': float(value[0]), 'lon': float(value[1])}
def get_gh_city(gh):
with open("google_geocity_cache.json","r") as f:
google_geocity_cache = json.load(f)
if gh in list(google_geocity_cache.keys()):
city = google_geocity_cache[gh]
else:
location = gmaps.reverse_geocode(geohash.decode(gh))
city = location[0]["formatted_address"].split(",")[1]
time.sleep(1)
google_geocity_cache[gh] = city
with open("google_geocity_cache.json","w") as f:
json.dump(google_geocity_cache,f)
time.sleep(1)
return city
def get_device_ids_by_geohash(self, geohash_value, meters):
""" get list of device_ids (UUID) based on distance from reference point
use haversine for accurate location since we should be iterating over small list of devices
"""
import geohash
import haversine
query = "SELECT device_uuid, geohash from devices"
try:
rows = self.session.execute(query)
self.log.info('We got %s rows' % len(rows))
except Exception:
self.log.exception('failed query execution.')
raise
point = geohash.decode(geohash_value)
# create list of devices whose distance is less than <meters> from geohash_value.
# first decode geohash to lat / long
# than calculated haversine distance between that point and target
devices = [row.device_uuid for row in rows
if haversine.haversine(point, geohash.decode(row.geohash)) <= meters]
return devices
def loc2geohash(self, loc):
"""
Used internally to convert the "\\*\\*loc" into a geohash.
:param latlon: E.g. (38.70, -90.29)
:param location: String to be be geocoded. Example "Sophienstr. 9, 10178 Berlin".
:param geohash: A geohash. Example: "u33dbczk"
:param who: Use the location of another pin.
"""
if 'latlon' in loc:
return geohash.encode(loc['latlon'],
precision=self._geohash_precision)
elif 'location' in loc:
cache_geocoding = (
loc['cache_geocoding'] if 'cache_geocoding' in loc
else self._cache_geocoding)
# take geohash from cache if geocoding caching is enabled
if cache_geocoding:
cache_key = hash_iter(('geocoding',
'google',
str(self._geohash_precision),
loc['location']))
gh = self._redis.get(cache_key)
if gh:
return gh
# geocode the location to a geohash
lat, lon = self._nominatim.geocode(loc['location'])
gh = geohash.encode(lat, lon,
precision=self._geohash_precision)
# if cache the geolocated location
if cache_geocoding:
self._redis.setex(cache_key, self._cache_geocoding, gh)
return gh
elif 'geohash' in loc:
return geohash.encode(*geohash.decode(loc['geohash']),
precision=self._geohash_precision)
elif 'who' in loc:
return self.geohash(loc['who'])
else:
raise TypeError('wrong location specificaton')
def decode(self, words):
"""Decode words back to latitude and longitude"""
words = words.split("-")
if len(words) == 3:
i = self.rugbits_to_int([self.three_wordlist.index(w) for w in words])
elif len(words) == 6:
i = self.bytes_to_int([self.six_wordlist.index(w) for w in words])
i = self.unpad(i)
else:
raise RuntimeError("Do not know how to decode a set of %i words."%(len(words)))
geo_hash = self.int_to_geo(i)
return geohash.decode(geo_hash)
def test_get_cells_in_circle(self):
"""Test geohashsearch.get_cells_in_circle
"""
lat = 42.6
lon = -5.7
distance = 100
precision = 9
cells = geohashsearch.get_cells_in_circle(lat, lon, 100, precision)
self.assertTrue(len(cells) > 0)
# The first cell should be the center.
self.assertEquals(geohash.encode(lat, lon, precision), cells[0])
# And every cell is in the circle.
for cell in cells:
(c_lat, c_lon) = geohash.decode(cell)
self.assertTrue(geohashsearch.distance(c_lat, c_lon, lat, lon) < distance)
def add_SC(self,node_gh,util_params = []):
if node_gh not in self.nodes():
node_data = {}
node_data['SC_index'] = int(self.node[self.newest_node()]["SC_index"]) + 1
node_data['GPS'] = geohash.decode(node_gh)
node_data['lat'] = node_data['GPS'][0]
node_data['lon'] = node_data['GPS'][1]
node_data['GPS_lon_lat'] = [node_data['lon'],node_data['lat']]
node_data['population'] = self.SC_population(node_gh)
node_data['geohash'] = node_gh
if util_params:
node_data['util_params'] = {self.util_attrbs[i]:util_params[i] for i in range(len(self.util_attrbs))}
self.add_node(node_gh,{key:node_data[key] for key in node_data.keys()})
self.add_connections(node_gh)
else:
print ("node already in network")
return self
def make_points_geohash(c1,c2,size): lat1,long1,latdelta,longdelta = geohash.decode_exactly(c1) lat2,long2 = geohash.decode(c2) latdelta,longdelta = latdelta * 2,longdelta * 2 # creating lats and longs longs = np.linspace(long1,long2,(abs(long2 - long1) / longdelta) + 1) lats = np.linspace(lat1,lat2,(abs(lat2 - lat1) / latdelta) + 1) total = [] count = 0 for long in longs: total += zip(lats,[long] * len(lats),[count] * len(lats)) count += 1 total = pd.DataFrame(total,columns=['LAT','LONG','X']) return map_table_new(total,precision=size)
def _kws2geohash(self, kws):
if 'latlon' in kws:
return geohash.encode(kws['latlon'],
precision=self._geohash_precision)
elif 'location' in kws:
cache_geocoding = (
kws['cache_geocoding'] if 'cache_geocoding' in kws
else self._cache_geocoding)
# take geohash from cache if geocoding caching is enabled
if cache_geocoding:
cache_key = hash_iter(('geocoding', 'google',
str(self._geohash_precision), kws['location']))
gh = self._redis.get(cache_key)
if gh:
return gh
# geocode the location to a geohash
lat, lon = self._nominatim.geocode(kws['location'])
gh = geohash.encode(lat, lon,
precision=self._geohash_precision)
# if cache the geolocated location
if cache_geocoding:
self._redis.setex(cache_key, self._cache_geocoding, gh)
return gh
elif 'geohash' in kws:
return geohash.encode(*geohash.decode(kws['geohash']),
precision=self._geohash_precision)
elif 'who' in kws:
return self.geohash(kws['who'])
else:
raise ValueError('needs one location specificaton')
def toGraph(self,heatThreshold=-float('Inf')):
validKeys = [h for h in self.countPerGeohash.keys() \
if self.countPerGeohash[h]>heatThreshold]
points = np.array([geohash.decode(h) for h in validKeys])
tri = Delaunay(points)
# neighbors = dict(zip(validKeys, [[] for x in validKeys]))
# print neighbors
graph = Graph(validKeys)
for t in tri.simplices:
for i in range(3):
mdist = gu.geohash_exactDistance(validKeys[t[i]],validKeys[t[(i+1)%3]])
try:
mweight = mdist/abs(self.countPerGeohash[validKeys[t[i]]]\
-self.countPerGeohash[validKeys[t[(i+1)%3]]])
except:
mweight = mdist
graph.addEdge(
(mdist,
self.countPerGeohash[validKeys[t[i]]]+self.countPerGeohash[validKeys[t[(i+1)%3]]],
validKeys[t[i]],validKeys[t[(i+1)%3]])
)
return graph
def streamEvents(gHash, userUuid): coords = geohash.decode(gHash) x = coords[0] * 1112.0 y = coords[1] * 1112.0 gHashes = hashesToSearch(x, y) return Response(eventStream(gHashes, userUuid), mimetype='text/event-stream')
def latlon(self, pin_id):
"""Return latitude and Longitude of a pin."""
gh = self.geohash(pin_id)
return geohash.decode(gh)
