def cli(assets_file):
"""Given a lighting alert for a specific area"""
with open(assets_file) as f:
assets_data = json.load(f)
assets_dict = {} # save the data as a dict format to improve execution efficiency
count_aline = 0
for asset in assets_data:
count_aline += 1
try:
if asset["quadKey"] not in assets_dict:
assets_dict[asset["quadKey"]] = [(asset["assetOwner"], asset["assetName"])]
else:
assets_dict[asset["quadKey"]].append((asset["assetOwner"], asset["assetName"]))
except Exception as e:
click.echo(click.style("Invalid asset input #%s: "% count_aline + str(asset),fg = 'red'))
"""Since quadkey might not be accurate enough
there might be more than one asset owners sharing a same quadkey
"""
count_lline = 0
for line in sys.stdin:
lightning_data = json.loads(line)
count_lline += 1
try:
if lightning_data["flashType"] == 9: # exclude flashType is 'heartbeat'
continue
latitude = lightning_data['latitude']
longitude = lightning_data['longitude']
qk = quadkey.from_geo((latitude, longitude), 12) # conver lightning data to quadkey
if qk.key in assets_dict:
asset_tuples = assets_dict[qk.key]
for asset in asset_tuples:
click.echo('lightning alert for %s : %s' % (asset[0], asset[1]))
del assets_dict[qk.key] # to prevent alerting several times
except Exception as e:
click.echo(click.style("Invalid strike input #%s: "% count_lline + str(line), fg = 'red'))
def run(self):
size_x, size_y = (IMG_X, IMG_Y)
conn = sqlite3.connect(self.output().fn)
cur = conn.cursor()
ddl = """CREATE TABLE IF NOT EXISTS {}(
qkey TEXT,
latitude FLOAT,
longtitude FLOAT,
ranking INTEGER,
hospital_id INTEGER,
duration FLOAT,
distance_path FLOAT
)""".format(self.table_name)
cur.execute(ddl)
ddl_index = "CREATE INDEX {}_index_qkey ON {}(qkey)".format(
self.table_name, self.table_name)
cur.execute(ddl_index)
dml = """INSERT INTO {}(
'qkey',
'latitude',
'longtitude',
'ranking',
'hospital_id',
'duration',
'distance_path')
VALUES (?, ?, ?, ?, ?, ?, ?)""".format(self.table_name)
for tile_data_file in self.input():
with open(tile_data_file.fn, 'r') as f:
result_data = pickle.load(f)
tile_x = result_data["tile_x"]
tile_y = result_data["tile_y"]
zoom = result_data["zoom"]
for img_y in range(0, size_y):
for img_x in range(0, size_x):
lat, lon = num2deg(tile_x + float(img_x) / size_x,
tile_y + float(img_y) / size_y, zoom)
qkey = quadkey.from_geo((lat, lon), QUADKEY_LEVEL).key
# 所要時間をDBに登録する
num = len(result_data["duration"][img_y][img_x])
for i in range(num):
try:
duration = result_data["duration"][img_y][img_x][
i]['value']
distance_path = result_data["distance_path"][
img_y][img_x][i]['value']
hospital_id = result_data["duration"][img_y][
img_x][i]['id']
cur.execute(dml, (qkey, lat, lon, i, hospital_id,
duration, distance_path))
except:
print "No distance / duration data at {}".format(i)
continue
conn.commit()
conn.close()
def GetQuadTiles(centerLat, centerLon, zoomLevel):
centerHash = quadkey.from_geo([centerLat, centerLon], zoomLevel)
qkeys = []
qkeys.append(centerHash.key)
for q in centerHash.nearby():
qkeys.append(q)
return qkeys
def nodes(self, nodes):
for osmid, tags, coord in nodes:
if 'name' in tags:
name = tags['name']
else:
name = None
if 'name:ja' in tags:
name = tags['name:ja']
lon, lat = coord
qkey = quadkey.from_geo((lat, lon), 16).key
cur.execute(nodes_dml, (osmid, name, qkey, lon, lat))
def update_stations():
data = download_ubike()
for k, v in data['retVal'].iteritems():
if not db.session.query(Station).filter(Station.sno == v['sno']).count():
reg = Station(v['sno'], v['sna'], v['lat'], v['lng'],
str(quadkey.from_geo((v['lat'], v['lng']), 17)), v['mday'])
db.session.add(reg)
db.session.commit()
updated = data['retVal']['0134']['mday']
return jsonify({'status': 'successed', 'updated': updated})
def toQuadkey_(self, input, export, sep = ',', level = 15):
toFile = open(export,'w')
with open(input) as f:
lines = f.readlines()
for line in lines:
name = line.split(sep)[0].strip()
addr = line.split(sep)[1].strip()
gq = GeocodeQuery("zh-tw", "tw")
gq.get_geocode(addr)
lat = gq.get_lat()
lng = gq.get_lng()
qk = quadkey.from_geo((lat, lng), level)
toFile.write(name + "," + addr + "," + str(lat) + "," + str(lng) + "," + str(qk.key) + '\n')
toFile.close()
def toQuadkey(self, df, col=2, level=15, geo='N'):
newDf = []
for row in df:
newRow = row
gq = GeocodeQuery("zh-tw", "tw")
gq.get_geocode(row[col - 1].encode('utf-8'))
lat = gq.get_lat()
lng = gq.get_lng()
qk = quadkey.from_geo((lat, lng), level)
newRow.append(qk.key)
if (geo == 'Y'):
newRow.extend([lat, lng])
newDf.append(newRow)
return newDf
def toQuadkey(self, df, col=2, level=15, geo = 'N'):
newDf = []
for row in df:
newRow = row
gq = GeocodeQuery("zh-tw", "tw")
gq.get_geocode(row[col-1].encode('utf-8'))
lat = gq.get_lat()
lng = gq.get_lng()
qk = quadkey.from_geo((lat, lng), level)
newRow.append(qk.key)
if (geo == 'Y'):
newRow.extend([lat, lng])
newDf.append(newRow)
return newDf
def toQuadkey_(self, input, export, sep=',', level=15):
toFile = open(export, 'w')
with open(input) as f:
lines = f.readlines()
for line in lines:
name = line.split(sep)[0].strip()
addr = line.split(sep)[1].strip()
gq = GeocodeQuery("zh-tw", "tw")
gq.get_geocode(addr)
lat = gq.get_lat()
lng = gq.get_lng()
qk = quadkey.from_geo((lat, lng), level)
toFile.write(name + "," + addr + "," + str(lat) + "," +
str(lng) + "," + str(qk.key) + '\n')
toFile.close()
def run(self):
tile_x = self.x
tile_y = self.y
zoom = self.zoom
size_x, size_y = (IMG_X, IMG_Y)
img = Image.new('RGBA', (size_x, size_y), (0, 0, 0, 0))
draw = ImageDraw.Draw(img)
# raw_data/generate_geohashindex.py で作成したDB
conn = sqlite3.connect(self.database, check_same_thread=False)
conn.row_factory = sqlite3.Row
cur = conn.cursor()
cur.execute("PRAGMA case_sensitive_like=ON;")
for img_y in range(0, size_y):
for img_x in range(0, size_x):
# 1ピクセル分
current_pos = num2deg(tile_x + float(img_x)/size_x, tile_y + float(img_y)/size_y, zoom)
qkey = quadkey.from_geo(current_pos, 16).key
#print qkey
try:
c = cur.execute(self.query.format(qkey))
row = c.fetchone()
#print (row['qkey'], row[self.row_name] , self.query.format(qkey))
value = row[self.row_name]
except:
#print (qkey, None)
value = self.null_value
if self.target_name == "population":
color = get_color_population(value)
else:
color = get_color(value)
draw.rectangle(((img_x,img_y),(img_x+1,img_y+1)),fill=color)
print "{} : {} / {}".format(self.img_file, img_y, size_y)
img = img.resize((TILE_SIZE, TILE_SIZE))
img.save(self.output().fn, 'PNG')
conn.close()
def add_location_info(record):
global reader
if reader is None:
reader = Reader(
"/home/cloudera/spark-2.4.4-bin-hadoop2.6/maxmind/GeoLite2-City.mmdb"
)
ip = record.split(",")[13]
try:
response = reader.city(ip)
country = response.country.name
city = response.city.name
latitude = response.location.latitude
longitude = response.location.longitude
qk = quadkey.from_geo((latitude, longitude), 15)
acc_num_good_records.add(1)
return "{},{},{},{},{},{}".format(record, country, city, latitude,
longitude, qk.key)
except:
acc_num_bad_records.add(1)
return "-----"
def find_stations(lat, lng):
dist = {}
result = []
unit = 14
while len(dist) < 2 and unit >= 5:
user_quadkey = str(quadkey.from_geo((lat, lng), unit))
logging.debug('user_quadkey: %s' % user_quadkey)
stations = Station.query.filter(Station.quadkey.like('%s%%' % user_quadkey)).all()
#print user_quadkey, stations
sno_candidate = [x.sno for x in stations]
sbis = Sbi.query.join(Station, Sbi.sno == Station.sno).\
add_columns(Sbi.sno, Sbi.act, Sbi.sbi, Station.sna, Station.lat, Station.lng).\
filter(Sbi.sno.in_(sno_candidate)).all()
logging.debug('sbis: %s', sbis)
for sbi in sbis:
if sbi.sbi > 0 and sbi.act > 0 :
detail = {}
detail['dist'] = haversine(float(lat), float(lng), sbi.lat, sbi.lng)
detail['sna'] = sbi.sna
detail['sbi'] = sbi.sbi
detail['act'] = sbi.act
dist[sbi.sno] = detail
logging.debug('dist: ' % dist)
if len(dist) >= 2:
tmp = sorted([i[1]['dist'] for i in dist.items()])[:2]
for k, v in dist.iteritems():
if v['dist'] in tmp:
result.append({'station':v['sna'], 'num_bike':v['sbi']})
else:
pass
unit -= 1
logging.debug('unit: %s' % unit)
return result
def coords(self, coords):
for osmid, lon, lat in coords:
qkey = quadkey.from_geo((lat, lon), 16).key
cur.execute(nodes_dml, (osmid, None, qkey, lon, lat))
def testFromGeo(self):
geo = (40, -105)
level = 7
key = quadkey.from_str('0231010')
self.assertEqual(key, quadkey.from_geo(geo, level))
def rows(self):
# 3次メッシュデータ読み込み
r = re.compile(r"[0-9]{8}")
stat_third_mesh = {}
with open(self.mesh_data, "r") as f:
for i, line in enumerate(f):
row = line.rstrip().split(',')
mesh_code = row[0]
if not r.match(mesh_code):
continue
value = float(row[self.value_row_number])
lat, lon = mesh_code_to_latlng(mesh_code)
stat_third_mesh[mesh_code] = [mesh_code, value, lat, lon]
# 緯度経度の秒数を求め、それぞれ最大最小を求める
area_lat_max = max(stat_third_mesh.values(), key=(lambda x: x[2]))[2]
area_lat_min = min(stat_third_mesh.values(), key=(lambda x: x[2]))[2]
area_lon_max = max(stat_third_mesh.values(), key=(lambda x: x[3]))[3]
area_lon_min = min(stat_third_mesh.values(), key=(lambda x: x[3]))[3]
lat_index_min = mesh_code_to_latlng_index(
min(stat_third_mesh.values(), key=(lambda x: x[2]))[0])[0]
lon_index_min = mesh_code_to_latlng_index(
min(stat_third_mesh.values(), key=(lambda x: x[3]))[0])[1]
#print (area_lat_max,area_lat_min,area_lon_max,area_lon_min)
# 位置計算関数作成 (度から秒に換算した上で割る, 1/2メッシュなので2を掛ける)
get_index_lat = lambda x: int(
(x - area_lat_min) * 3600.0 * 2.0 / 30.0 + 1.0
) # quadkeyの視点は北西、1/2メッシュの視点は南西なのでインデックスがずれる
get_index_lon = lambda x: int((x - area_lon_min) * 3600.0 * 2.0 / 45.0)
# メッシュ数計算
mesh_nums = (get_index_lat(area_lat_max) + 1,
get_index_lon(area_lon_max) + 1)
#print mesh_nums
# 統計値格納配列を準備する
array_third_mesh = np.zeros(mesh_nums[0] * mesh_nums[1]).reshape(
mesh_nums[0], mesh_nums[1])
# 統計値代入
for x in stat_third_mesh.values():
current_mesh_code = x[0]
lat_index, lon_index = mesh_code_to_latlng_index(x[0])
array_third_mesh[lat_index - lat_index_min,
lon_index - lon_index_min] = x[1]
if (len(current_mesh_code) == 8):
array_third_mesh[lat_index - lat_index_min - 1,
lon_index - lon_index_min] = x[1]
array_third_mesh[lat_index - lat_index_min,
lon_index - lon_index_min - 1] = x[1]
array_third_mesh[lat_index - lat_index_min - 1,
lon_index - lon_index_min - 1] = x[1]
draw_matrix(array_third_mesh,
'./var/T00_mesh_code_map_{}.png'.format(self.table_name))
# quadkey空間におけるピクセル数算出
zoom = 8
pixel_ne = deg_to_pixel_coordinates(area_lat_max, area_lon_max, zoom)
pixel_sw = deg_to_pixel_coordinates(area_lat_min, area_lon_min, zoom)
pixel_nums = (pixel_sw[1] - pixel_ne[1] + 1,
pixel_ne[0] - pixel_sw[0] + 1) # h*w
print pixel_nums
# リサンプリング比率計算
resampling_ratio = (float(pixel_nums[0]) / float(mesh_nums[0] - 1),
float(pixel_nums[1]) / float(mesh_nums[1] - 1))
print resampling_ratio
# リサンプリング実行
print array_third_mesh.sum()
resampled_mesh = scipy.ndimage.zoom(array_third_mesh,
resampling_ratio,
order=0)
# ゴミをゼロに丸める
value_limit = 0.001 # 0.01 / resampling_ratio[0] / resampling_ratio[1]
#resampled_mesh = resampled_mesh * (resampled_mesh < value_limit)
if self.no_unit_value:
# 単位のない値(割合等)は値を正規化しない
quadkey_mesh = resampled_mesh
else:
# 値を正規化する
quadkey_mesh = resampled_mesh * array_third_mesh.sum(
) / resampled_mesh.sum()
# リサンプル後のメッシュデータのインデックスから緯度経度を計算する関数
index_to_deg = lambda i, j: (
float(i + 1) / pixel_nums[0] * (area_lat_max - area_lat_min) +
area_lat_min, float(j + 1) / pixel_nums[1] *
(area_lon_max - area_lon_min) + area_lon_min)
sum_value = 0
# リサンプル後のメッシュデータの各要素をループしてDBに保存
qkeys = []
for lat_index, row in enumerate(quadkey_mesh):
for lon_index, value in enumerate(row):
if value < value_limit:
continue
else:
sum_value += value
lat, lon = index_to_deg(lat_index, lon_index)
qkey = quadkey.from_geo((lat, lon), 16).key
exists = qkey in qkeys
qkeys.append(qkey)
try:
yield (qkey, lat, lon, value)
except:
continue
print sum_value
def quadkeyfromxy(x, y, zoom):
lat, lon = latlonfromxy(x, y, zoom)
qk = quadkey.from_geo((lat, lon), zoom)
return qk.key
def quadkeyfromlatlon(lat_deg, lon_deg, zoom):
qk = quadkey.from_geo((lat, lon), zoom)
return qk.key
def testFromGeo(self):
geo = (40, -105)
level = 7
key = quadkey.from_str('0231010')
self.assertEqual(key, quadkey.from_geo(geo, level))
def run(self):
tile_x = self.x
tile_y = self.y
zoom = self.zoom
size_x, size_y = (IMG_X, IMG_Y)
target_list = self.hospitals
conn = sqlite3.connect(self.input().connection_string.replace(
"sqlite:///", ""))
cur = conn.cursor()
result_data = {
"zoom":
zoom,
"tile_x":
tile_x,
"tile_y":
tile_y,
"distance_straight_line":
[[{} for i in range(size_x)] for j in range(size_y)],
"distance_path":
[[{} for i in range(size_x)] for j in range(size_y)],
"duration": [[{} for i in range(size_x)] for j in range(size_y)]
}
for img_y in range(0, size_y):
for img_x in range(0, size_x):
print "{} : {}, {} / {}, {}".format(self.output().fn, img_x,
img_y, size_x, size_y)
# 1ピクセル分
current_pos = num2deg(tile_x + float(img_x) / size_x,
tile_y + float(img_y) / size_y, zoom)
## 人口メッシュにデータが有る場合のみ続行する
qkey = quadkey.from_geo(current_pos, QUADKEY_LEVEL).key
cur.execute('select qkey from mesh_population where qkey = ?',
(qkey, ))
if cur.fetchone() == None:
continue
result = search_and_sort_places_by_distance(*(current_pos +
(target_list, )))
result_data["distance_straight_line"][img_y][img_x] = [{
'id':
x['id'],
'value':
x['distance']
} for x in result]
result = search_and_sort_places_by_duration(
*(current_pos + ([
x for x in target_list
if x['distance'] <= self.distance_threshold
], ))) # 近いものに絞って時間距離を求める
result_data["duration"][img_y][img_x] = [{
'id': x['id'],
'value': x['duration']
} for x in result]
result_data["distance_path"][img_y][img_x] = [{
'id':
x['id'],
'value':
x['distance']
} for x in result]
with self.output().open('w') as f:
pickle.dump(result_data, f, pickle.HIGHEST_PROTOCOL)
conn.close()