def cli(assets_file): """Given a lighting alert for a specific area""" with open(assets_file) as f: assets_data = json.load(f) assets_dict = {} # save the data as a dict format to improve execution efficiency count_aline = 0 for asset in assets_data: count_aline += 1 try: if asset["quadKey"] not in assets_dict: assets_dict[asset["quadKey"]] = [(asset["assetOwner"], asset["assetName"])] else: assets_dict[asset["quadKey"]].append((asset["assetOwner"], asset["assetName"])) except Exception as e: click.echo(click.style("Invalid asset input #%s: "% count_aline + str(asset),fg = 'red')) """Since quadkey might not be accurate enough there might be more than one asset owners sharing a same quadkey """ count_lline = 0 for line in sys.stdin: lightning_data = json.loads(line) count_lline += 1 try: if lightning_data["flashType"] == 9: # exclude flashType is 'heartbeat' continue latitude = lightning_data['latitude'] longitude = lightning_data['longitude'] qk = quadkey.from_geo((latitude, longitude), 12) # conver lightning data to quadkey if qk.key in assets_dict: asset_tuples = assets_dict[qk.key] for asset in asset_tuples: click.echo('lightning alert for %s : %s' % (asset[0], asset[1])) del assets_dict[qk.key] # to prevent alerting several times except Exception as e: click.echo(click.style("Invalid strike input #%s: "% count_lline + str(line), fg = 'red'))
def run(self): size_x, size_y = (IMG_X, IMG_Y) conn = sqlite3.connect(self.output().fn) cur = conn.cursor() ddl = """CREATE TABLE IF NOT EXISTS {}( qkey TEXT, latitude FLOAT, longtitude FLOAT, ranking INTEGER, hospital_id INTEGER, duration FLOAT, distance_path FLOAT )""".format(self.table_name) cur.execute(ddl) ddl_index = "CREATE INDEX {}_index_qkey ON {}(qkey)".format( self.table_name, self.table_name) cur.execute(ddl_index) dml = """INSERT INTO {}( 'qkey', 'latitude', 'longtitude', 'ranking', 'hospital_id', 'duration', 'distance_path') VALUES (?, ?, ?, ?, ?, ?, ?)""".format(self.table_name) for tile_data_file in self.input(): with open(tile_data_file.fn, 'r') as f: result_data = pickle.load(f) tile_x = result_data["tile_x"] tile_y = result_data["tile_y"] zoom = result_data["zoom"] for img_y in range(0, size_y): for img_x in range(0, size_x): lat, lon = num2deg(tile_x + float(img_x) / size_x, tile_y + float(img_y) / size_y, zoom) qkey = quadkey.from_geo((lat, lon), QUADKEY_LEVEL).key # 所要時間をDBに登録する num = len(result_data["duration"][img_y][img_x]) for i in range(num): try: duration = result_data["duration"][img_y][img_x][ i]['value'] distance_path = result_data["distance_path"][ img_y][img_x][i]['value'] hospital_id = result_data["duration"][img_y][ img_x][i]['id'] cur.execute(dml, (qkey, lat, lon, i, hospital_id, duration, distance_path)) except: print "No distance / duration data at {}".format(i) continue conn.commit() conn.close()
def GetQuadTiles(centerLat, centerLon, zoomLevel): centerHash = quadkey.from_geo([centerLat, centerLon], zoomLevel) qkeys = [] qkeys.append(centerHash.key) for q in centerHash.nearby(): qkeys.append(q) return qkeys
def nodes(self, nodes): for osmid, tags, coord in nodes: if 'name' in tags: name = tags['name'] else: name = None if 'name:ja' in tags: name = tags['name:ja'] lon, lat = coord qkey = quadkey.from_geo((lat, lon), 16).key cur.execute(nodes_dml, (osmid, name, qkey, lon, lat))
def update_stations(): data = download_ubike() for k, v in data['retVal'].iteritems(): if not db.session.query(Station).filter(Station.sno == v['sno']).count(): reg = Station(v['sno'], v['sna'], v['lat'], v['lng'], str(quadkey.from_geo((v['lat'], v['lng']), 17)), v['mday']) db.session.add(reg) db.session.commit() updated = data['retVal']['0134']['mday'] return jsonify({'status': 'successed', 'updated': updated})
def toQuadkey_(self, input, export, sep = ',', level = 15): toFile = open(export,'w') with open(input) as f: lines = f.readlines() for line in lines: name = line.split(sep)[0].strip() addr = line.split(sep)[1].strip() gq = GeocodeQuery("zh-tw", "tw") gq.get_geocode(addr) lat = gq.get_lat() lng = gq.get_lng() qk = quadkey.from_geo((lat, lng), level) toFile.write(name + "," + addr + "," + str(lat) + "," + str(lng) + "," + str(qk.key) + '\n') toFile.close()
def toQuadkey(self, df, col=2, level=15, geo='N'): newDf = [] for row in df: newRow = row gq = GeocodeQuery("zh-tw", "tw") gq.get_geocode(row[col - 1].encode('utf-8')) lat = gq.get_lat() lng = gq.get_lng() qk = quadkey.from_geo((lat, lng), level) newRow.append(qk.key) if (geo == 'Y'): newRow.extend([lat, lng]) newDf.append(newRow) return newDf
def toQuadkey(self, df, col=2, level=15, geo = 'N'): newDf = [] for row in df: newRow = row gq = GeocodeQuery("zh-tw", "tw") gq.get_geocode(row[col-1].encode('utf-8')) lat = gq.get_lat() lng = gq.get_lng() qk = quadkey.from_geo((lat, lng), level) newRow.append(qk.key) if (geo == 'Y'): newRow.extend([lat, lng]) newDf.append(newRow) return newDf
def toQuadkey_(self, input, export, sep=',', level=15): toFile = open(export, 'w') with open(input) as f: lines = f.readlines() for line in lines: name = line.split(sep)[0].strip() addr = line.split(sep)[1].strip() gq = GeocodeQuery("zh-tw", "tw") gq.get_geocode(addr) lat = gq.get_lat() lng = gq.get_lng() qk = quadkey.from_geo((lat, lng), level) toFile.write(name + "," + addr + "," + str(lat) + "," + str(lng) + "," + str(qk.key) + '\n') toFile.close()
def run(self): tile_x = self.x tile_y = self.y zoom = self.zoom size_x, size_y = (IMG_X, IMG_Y) img = Image.new('RGBA', (size_x, size_y), (0, 0, 0, 0)) draw = ImageDraw.Draw(img) # raw_data/generate_geohashindex.py で作成したDB conn = sqlite3.connect(self.database, check_same_thread=False) conn.row_factory = sqlite3.Row cur = conn.cursor() cur.execute("PRAGMA case_sensitive_like=ON;") for img_y in range(0, size_y): for img_x in range(0, size_x): # 1ピクセル分 current_pos = num2deg(tile_x + float(img_x)/size_x, tile_y + float(img_y)/size_y, zoom) qkey = quadkey.from_geo(current_pos, 16).key #print qkey try: c = cur.execute(self.query.format(qkey)) row = c.fetchone() #print (row['qkey'], row[self.row_name] , self.query.format(qkey)) value = row[self.row_name] except: #print (qkey, None) value = self.null_value if self.target_name == "population": color = get_color_population(value) else: color = get_color(value) draw.rectangle(((img_x,img_y),(img_x+1,img_y+1)),fill=color) print "{} : {} / {}".format(self.img_file, img_y, size_y) img = img.resize((TILE_SIZE, TILE_SIZE)) img.save(self.output().fn, 'PNG') conn.close()
def add_location_info(record): global reader if reader is None: reader = Reader( "/home/cloudera/spark-2.4.4-bin-hadoop2.6/maxmind/GeoLite2-City.mmdb" ) ip = record.split(",")[13] try: response = reader.city(ip) country = response.country.name city = response.city.name latitude = response.location.latitude longitude = response.location.longitude qk = quadkey.from_geo((latitude, longitude), 15) acc_num_good_records.add(1) return "{},{},{},{},{},{}".format(record, country, city, latitude, longitude, qk.key) except: acc_num_bad_records.add(1) return "-----"
def find_stations(lat, lng): dist = {} result = [] unit = 14 while len(dist) < 2 and unit >= 5: user_quadkey = str(quadkey.from_geo((lat, lng), unit)) logging.debug('user_quadkey: %s' % user_quadkey) stations = Station.query.filter(Station.quadkey.like('%s%%' % user_quadkey)).all() #print user_quadkey, stations sno_candidate = [x.sno for x in stations] sbis = Sbi.query.join(Station, Sbi.sno == Station.sno).\ add_columns(Sbi.sno, Sbi.act, Sbi.sbi, Station.sna, Station.lat, Station.lng).\ filter(Sbi.sno.in_(sno_candidate)).all() logging.debug('sbis: %s', sbis) for sbi in sbis: if sbi.sbi > 0 and sbi.act > 0 : detail = {} detail['dist'] = haversine(float(lat), float(lng), sbi.lat, sbi.lng) detail['sna'] = sbi.sna detail['sbi'] = sbi.sbi detail['act'] = sbi.act dist[sbi.sno] = detail logging.debug('dist: ' % dist) if len(dist) >= 2: tmp = sorted([i[1]['dist'] for i in dist.items()])[:2] for k, v in dist.iteritems(): if v['dist'] in tmp: result.append({'station':v['sna'], 'num_bike':v['sbi']}) else: pass unit -= 1 logging.debug('unit: %s' % unit) return result
def coords(self, coords): for osmid, lon, lat in coords: qkey = quadkey.from_geo((lat, lon), 16).key cur.execute(nodes_dml, (osmid, None, qkey, lon, lat))
def testFromGeo(self): geo = (40, -105) level = 7 key = quadkey.from_str('0231010') self.assertEqual(key, quadkey.from_geo(geo, level))
def rows(self): # 3次メッシュデータ読み込み r = re.compile(r"[0-9]{8}") stat_third_mesh = {} with open(self.mesh_data, "r") as f: for i, line in enumerate(f): row = line.rstrip().split(',') mesh_code = row[0] if not r.match(mesh_code): continue value = float(row[self.value_row_number]) lat, lon = mesh_code_to_latlng(mesh_code) stat_third_mesh[mesh_code] = [mesh_code, value, lat, lon] # 緯度経度の秒数を求め、それぞれ最大最小を求める area_lat_max = max(stat_third_mesh.values(), key=(lambda x: x[2]))[2] area_lat_min = min(stat_third_mesh.values(), key=(lambda x: x[2]))[2] area_lon_max = max(stat_third_mesh.values(), key=(lambda x: x[3]))[3] area_lon_min = min(stat_third_mesh.values(), key=(lambda x: x[3]))[3] lat_index_min = mesh_code_to_latlng_index( min(stat_third_mesh.values(), key=(lambda x: x[2]))[0])[0] lon_index_min = mesh_code_to_latlng_index( min(stat_third_mesh.values(), key=(lambda x: x[3]))[0])[1] #print (area_lat_max,area_lat_min,area_lon_max,area_lon_min) # 位置計算関数作成 (度から秒に換算した上で割る, 1/2メッシュなので2を掛ける) get_index_lat = lambda x: int( (x - area_lat_min) * 3600.0 * 2.0 / 30.0 + 1.0 ) # quadkeyの視点は北西、1/2メッシュの視点は南西なのでインデックスがずれる get_index_lon = lambda x: int((x - area_lon_min) * 3600.0 * 2.0 / 45.0) # メッシュ数計算 mesh_nums = (get_index_lat(area_lat_max) + 1, get_index_lon(area_lon_max) + 1) #print mesh_nums # 統計値格納配列を準備する array_third_mesh = np.zeros(mesh_nums[0] * mesh_nums[1]).reshape( mesh_nums[0], mesh_nums[1]) # 統計値代入 for x in stat_third_mesh.values(): current_mesh_code = x[0] lat_index, lon_index = mesh_code_to_latlng_index(x[0]) array_third_mesh[lat_index - lat_index_min, lon_index - lon_index_min] = x[1] if (len(current_mesh_code) == 8): array_third_mesh[lat_index - lat_index_min - 1, lon_index - lon_index_min] = x[1] array_third_mesh[lat_index - lat_index_min, lon_index - lon_index_min - 1] = x[1] array_third_mesh[lat_index - lat_index_min - 1, lon_index - lon_index_min - 1] = x[1] draw_matrix(array_third_mesh, './var/T00_mesh_code_map_{}.png'.format(self.table_name)) # quadkey空間におけるピクセル数算出 zoom = 8 pixel_ne = deg_to_pixel_coordinates(area_lat_max, area_lon_max, zoom) pixel_sw = deg_to_pixel_coordinates(area_lat_min, area_lon_min, zoom) pixel_nums = (pixel_sw[1] - pixel_ne[1] + 1, pixel_ne[0] - pixel_sw[0] + 1) # h*w print pixel_nums # リサンプリング比率計算 resampling_ratio = (float(pixel_nums[0]) / float(mesh_nums[0] - 1), float(pixel_nums[1]) / float(mesh_nums[1] - 1)) print resampling_ratio # リサンプリング実行 print array_third_mesh.sum() resampled_mesh = scipy.ndimage.zoom(array_third_mesh, resampling_ratio, order=0) # ゴミをゼロに丸める value_limit = 0.001 # 0.01 / resampling_ratio[0] / resampling_ratio[1] #resampled_mesh = resampled_mesh * (resampled_mesh < value_limit) if self.no_unit_value: # 単位のない値(割合等)は値を正規化しない quadkey_mesh = resampled_mesh else: # 値を正規化する quadkey_mesh = resampled_mesh * array_third_mesh.sum( ) / resampled_mesh.sum() # リサンプル後のメッシュデータのインデックスから緯度経度を計算する関数 index_to_deg = lambda i, j: ( float(i + 1) / pixel_nums[0] * (area_lat_max - area_lat_min) + area_lat_min, float(j + 1) / pixel_nums[1] * (area_lon_max - area_lon_min) + area_lon_min) sum_value = 0 # リサンプル後のメッシュデータの各要素をループしてDBに保存 qkeys = [] for lat_index, row in enumerate(quadkey_mesh): for lon_index, value in enumerate(row): if value < value_limit: continue else: sum_value += value lat, lon = index_to_deg(lat_index, lon_index) qkey = quadkey.from_geo((lat, lon), 16).key exists = qkey in qkeys qkeys.append(qkey) try: yield (qkey, lat, lon, value) except: continue print sum_value
def quadkeyfromxy(x, y, zoom): lat, lon = latlonfromxy(x, y, zoom) qk = quadkey.from_geo((lat, lon), zoom) return qk.key
def quadkeyfromlatlon(lat_deg, lon_deg, zoom): qk = quadkey.from_geo((lat, lon), zoom) return qk.key
def testFromGeo(self): geo = (40, -105) level = 7 key = quadkey.from_str('0231010') self.assertEqual(key, quadkey.from_geo(geo, level))
def run(self): tile_x = self.x tile_y = self.y zoom = self.zoom size_x, size_y = (IMG_X, IMG_Y) target_list = self.hospitals conn = sqlite3.connect(self.input().connection_string.replace( "sqlite:///", "")) cur = conn.cursor() result_data = { "zoom": zoom, "tile_x": tile_x, "tile_y": tile_y, "distance_straight_line": [[{} for i in range(size_x)] for j in range(size_y)], "distance_path": [[{} for i in range(size_x)] for j in range(size_y)], "duration": [[{} for i in range(size_x)] for j in range(size_y)] } for img_y in range(0, size_y): for img_x in range(0, size_x): print "{} : {}, {} / {}, {}".format(self.output().fn, img_x, img_y, size_x, size_y) # 1ピクセル分 current_pos = num2deg(tile_x + float(img_x) / size_x, tile_y + float(img_y) / size_y, zoom) ## 人口メッシュにデータが有る場合のみ続行する qkey = quadkey.from_geo(current_pos, QUADKEY_LEVEL).key cur.execute('select qkey from mesh_population where qkey = ?', (qkey, )) if cur.fetchone() == None: continue result = search_and_sort_places_by_distance(*(current_pos + (target_list, ))) result_data["distance_straight_line"][img_y][img_x] = [{ 'id': x['id'], 'value': x['distance'] } for x in result] result = search_and_sort_places_by_duration( *(current_pos + ([ x for x in target_list if x['distance'] <= self.distance_threshold ], ))) # 近いものに絞って時間距離を求める result_data["duration"][img_y][img_x] = [{ 'id': x['id'], 'value': x['duration'] } for x in result] result_data["distance_path"][img_y][img_x] = [{ 'id': x['id'], 'value': x['distance'] } for x in result] with self.output().open('w') as f: pickle.dump(result_data, f, pickle.HIGHEST_PROTOCOL) conn.close()