def near_endpoint(t0, t1):
thread = 100
x0, y0, x1, y1 = t0
_x0, _y0, _x1, _y1 = t1
d0, d1 = calc_dist([x0, y0], [_x0, _y0]), calc_dist([x1, y1], [_x1, _y1])
_d0, _d1 = calc_dist([x0, y0], [_x1, _y1]), calc_dist([x1, y1], [_x0, _y0])
return d0 < thread and d1 < thread or _d0 < thread and _d1 < thread
def dst():
conn = oracle_util.get_connection()
cursor = conn.cursor()
sql = "select * from tb_road_point order by rid, seq"
cursor.execute(sql)
road = [[] for x in range(5432)]
for item in cursor:
lng, lat = map(float, item[2:4])
rid, seq = map(int, item[0:2])
px, py = bl2xy(lat, lng)
road[rid].append([px, py])
tup_list = []
upd_sql = "update tb_road_state set road_desc = :1 where rid = :2"
for rid in range(5432):
last_point = None
tot = 0
for point in road[rid]:
if last_point is not None:
tot += calc_dist(last_point, point)
last_point = point
tup_list.append((tot, rid))
cursor.executemany(upd_sql, tup_list)
conn.commit()
cursor.close()
conn.close()
def store_edge(tree):
p = tree.find('meta')
wys = p.findall('way')
for w in wys:
way_dic = w.attrib
wid = way_dic['id']
node_list = w.findall('nd')
map_way[wid] = {}
oneway = False
ref = map_way[wid]
tag_list = w.findall('tag')
for tag in tag_list:
tag_dic = tag.attrib
ref[tag_dic['k']] = tag_dic['v']
if 'oneway' in ref:
oneway = ref['oneway'] == 'yes'
node_in_way = []
for nd in node_list:
node_dic = nd.attrib
node_in_way.append(map_node_dict[node_dic['ref']])
ref['node'] = node_in_way
last_node = None
ref['edge'] = []
for node in node_in_way:
if last_node is not None:
edge_index = len(map_edge_list)
ref['edge'].append(edge_index)
p0, p1 = last_node.point, node.point
edge_length = calc_dist(p0, p1)
edge = MapEdge(last_node, node, oneway, edge_index, edge_length, wid)
map_edge_list.append(edge)
last_node = node
def center_5():
"""
修正曲折道路
"""
road_data = load_model('./road/center.txt')
mod_road = {51}
new_road_data = []
for road in road_data:
polyline = road['polyline']
name = road['name']
rid = int(road['rid'])
if rid in mod_road:
print name, rid, 'modified'
else:
new_road_data.append(road)
continue
xy_list = polyline2xylist(polyline)
new_xy_list = []
# 和2的不同是，确定最后一个点是正确的点
# 一旦越界，则不再继续
last_xy, final_xy = xy_list[0], xy_list[-1]
first_xy = last_xy[:]
final_dist = calc_dist(last_xy, final_xy)
new_xy_list.append(last_xy)
del xy_list[0]
del xy_list[-1]
len_list = len(xy_list)
for i in range(len_list - 1):
min_dist, sel = 1e10, -1
for j, xy in enumerate(xy_list):
dist = calc_dist(last_xy, xy)
if dist < min_dist:
min_dist, sel = dist, j
off_dist = calc_dist(xy_list[sel], first_xy)
if off_dist > final_dist:
break
last_xy = xy_list[sel]
new_xy_list.append(last_xy)
del xy_list[sel]
del new_xy_list[7]
new_xy_list.append(final_xy)
road['polyline'] = xylist2polyline(new_xy_list)
new_road_data.append(road)
save_model('./road/center_.txt', new_road_data)
def get_st_score(point, last_point, cur_edge, last_edge):
_, trace_dist = get_trace_dyn(last_edge, cur_edge, last_point, point)
p0, p1 = cur_edge.node0.point, cur_edge.node1.point
dist = point2segment(point, p0, p1)
dir_dist = calc_dist(last_point, point)
pro_d = get_guass_proc(dist)
pro_f = dir_dist / dist
score = -math.log(pro_d * pro_f + 1)
return score, dist
def get_trace_dist(trace):
last_point = None
trace_dist = 0
for point in trace:
if last_point is not None:
dist = calc_dist(point, last_point)
trace_dist += dist
last_point = point
return trace_dist
def trans():
tunnel = np.loadtxt('../data/tunnel.txt', delimiter=',')
dist_mat = np.loadtxt('./data/dist.txt')
n, _ = dist_mat.shape[:]
tup_list = []
for i in range(n):
p0, p1, p2, p3 = tunnel[i][:]
if calc_dist([p0, p1], [p2, p3]) < 1000:
continue
for j in range(i + 1, n):
p0, p1, p2, p3 = tunnel[j][:]
if calc_dist([p0, p1], [p2, p3]) < 1000:
continue
if dist_mat[j][i] < 50 and dist_mat[i][j] < 50:
tup_list.append("{0},{1},{2:.2f}\n".format(
i, j, dist_mat[i][j]))
np.savetxt('./data/parse.txt', tup_list, fmt='%s')
def center_1():
"""
某些道路制作过度，center1线出现折返情况
按照最左点为第一个点
:return:
"""
road_data = load_model('./road/center1.txt')
mod_road = {
39, 32 / 2, 58 / 2, 62 / 2, 196 / 2, 234 / 2, 292 / 2, 1016 / 2,
1112 / 2
}
new_road_data = []
# 修改
for road in road_data:
polyline = road['polyline']
name = road['name']
rid = road['rid']
if rid not in mod_road:
new_road_data.append(road)
continue
xy_list = polyline2xylist(polyline)
new_xy_list = []
# 对每个点找出最接近的那个点
min_x = 1e10
for x, y in xy_list:
if x < min_x:
min_x, last_xy = x, [x, y]
# last_xy = xy_list[0]
# new_xy_list.append(last_xy)
# del xy_list[0]
len_list = len(xy_list)
for i in range(len_list):
min_dist, sel = 1e10, -1
for j, xy in enumerate(xy_list):
dist = calc_dist(last_xy, xy)
if dist < min_dist:
min_dist, sel = dist, j
last_xy = xy_list[sel]
new_xy_list.append(last_xy)
del xy_list[sel]
new_xy_list = reversed(new_xy_list)
# 去重
temp_list = []
last_xy = [0, 0]
for x, y in new_xy_list:
if x == last_xy[0] and y == last_xy[1]:
print 'duplicated', name
continue
temp_list.append([x, y])
last_xy = [x, y]
new_xy_list = temp_list
road['polyline'] = xylist2polyline(new_xy_list)
new_road_data.append(road)
save_model('./road/center1.txt', new_road_data)
def load_map(self, map_level):
conn = oracle_util.get_connection()
sql = "select * from tb_road_state where map_level = :1"
tup = (map_level, )
cursor = conn.cursor()
cursor.execute(sql, tup)
for item in cursor:
road_name, direction, level = item[1:4]
r = MapRoad(road_name, direction, level)
rid = item[0]
self.map_road[rid] = r
map_temp_node = {} # 维护地图节点
road_point = {}
sql = "select rp.* from tb_road_point rp, tb_road_state" \
" rs where rp.rid = rs.rid and rs.map_level = :1 and rp.map_level = :2 order by rp.rid, rp.seq "
tup = (map_level, map_level)
cursor.execute(sql, tup)
for item in cursor:
rid = item[0]
lng, lat = map(float, item[2:4])
try:
road_point[rid].append([lng, lat])
except KeyError:
road_point[rid] = [[lng, lat]]
# 维护节点dict
# 每一个点都单独列入dict，边用两点表示，道路用若干点表示
for rid, bllist in road_point.iteritems():
r = self.map_road[rid]
last_nodeid = -1
lastx, lasty = None, None
for i, bl in enumerate(bllist):
lng, lat = bl[0:2]
str_bl = "{0},{1}".format(lng, lat)
x, y = bl2xy(lat, lng)
try:
nodeid = map_temp_node[str_bl]
nd = self.map_node[nodeid]
except KeyError:
nodeid = len(map_temp_node)
map_temp_node[str_bl] = nodeid
nd = MapNode([x, y], nodeid)
self.map_node[nodeid] = nd
if i > 0:
edge_length = calc_dist([x, y], [lastx, lasty])
edge = MapEdge(self.map_node[last_nodeid],
self.map_node[nodeid], True,
len(self.map_edge), edge_length, rid)
self.map_edge.append(edge)
r.add_node(nd)
last_nodeid, lastx, lasty = nodeid, x, y
conn.close()
def get_candidate_later(cur_data, last_data, last_point, last_edge, last_state,
itv_time, cnt):
"""
:param cur_data: current taxi_data
:param last_data: last taxi_data
:param last_point
:param last_edge: MapEdge
:param last_state: direction of vehicle in map edge
:return: edge_can_list [edge0, edge1....]
"""
edge_can_list = []
node_set = set() # node_set用于判断是否访问过
edge_set = set() # edge_set用于记录能够访问到的边
cur_point = [cur_data.px, cur_data.py]
cart_dist = calc_dist(cur_point, last_point)
eva_speed = max(cur_data.speed, last_data.speed) / 1.8
if itv_time > 60:
T = min(2.0 * cart_dist, 60 / 3.6 * itv_time)
else:
T = max(3.0 * cart_dist, eva_speed * itv_time) # dist_thread
# print 'thread', cnt, T, eva_speed, cart_dist, itv_time
if last_edge.oneway is False or is_near_segment(last_point, cur_point,
last_edge.node0.point,
last_edge.node1.point):
edge_set.add(last_edge.edge_index)
q = Queue.PriorityQueue(maxsize=-1) # 优先队列 best first search
init_candidate_queue(last_point, last_edge, q, node_set) # 搜索第一步，加入之前线段中的点
while not q.empty():
dnode = q.get()
cur_node, cur_dist = dnode.node, dnode.dist
if cur_dist >= T: # 超过阈值后停止
break
for edge, node in cur_node.link_list:
if node.nodeid in node_set:
continue
node_set.add(node.nodeid)
# 单行线需要判断角度
if edge.oneway is False or is_near_segment(
last_point, cur_point, edge.node0.point, edge.node1.point):
edge_set.add(edge.edge_index)
next_dnode = DistNode(node, cur_dist + edge.edge_length)
node.prev_node = cur_node
q.put(next_dnode)
for i in edge_set:
edge = map_edge_list[i]
dist = point2segment(cur_point, edge.node0.point, edge.node1.point)
if dist < 60:
edge_can_list.append(map_edge_list[i])
return edge_can_list
def merge(new_line, xy_list):
in_idx = -1
for i, new_pt in enumerate(new_line):
last_pt = None
min_dist = 1e10
for pt in xy_list:
if last_pt is not None:
dist = point2segment_xy(new_pt, last_pt, pt)
min_dist = min(dist, min_dist)
last_pt = pt
# print min_dist
if min_dist < 50 and in_idx == -1:
in_idx = i
new_line = new_line[:in_idx]
new_pt = new_line[-1]
dist0, dist1 = calc_dist(new_pt,
xy_list[0]), calc_dist(new_pt, xy_list[-1])
if dist0 < dist1:
new_line = np.append(new_line, [xy_list[0]], axis=0)
else:
new_line = np.append(new_line, [xy_list[-1]], axis=0)
return new_line
def get_gps_data(conn, begin_time, veh):
str_bt = begin_time.strftime('%Y-%m-%d %H:%M:%S')
end_time = begin_time + timedelta(days=1)
str_et = end_time.strftime('%Y-%m-%d %H:%M:%S')
sql = "select px, py, speed_time, state, speed, carstate, direction from " \
"TB_GPS_1709 t where speed_time >= to_date('{1}', 'yyyy-mm-dd hh24:mi:ss') " \
"and speed_time < to_date('{2}', 'yyyy-MM-dd hh24:mi:ss')" \
" and vehicle_num = '{0}'".format(veh, str_bt, str_et)
cursor = conn.cursor()
cursor.execute(sql)
trace = []
last_point = None
for item in cursor.fetchall():
lng, lat = map(float, item[0:2])
if 119 < lng < 121 and 29 < lat < 31:
px, py = bl2xy(lat, lng)
state = int(item[3])
stime = item[2]
speed = float(item[4])
carstate = int(item[5])
dir = float(item[6])
taxi_data = TaxiData(px, py, stime, state, speed, carstate, dir)
trace.append(taxi_data)
# print len(trace)
trace.sort(cmp1)
new_trace = []
for data in trace:
cur_point = data
if last_point is not None:
dist = calc_dist([cur_point.px, cur_point.py], [last_point.px, last_point.py])
del_time = (cur_point.stime - last_point.stime).total_seconds()
if data.speed > 140 or del_time < 5: # 速度阈值 时间间隔阈值
continue
elif dist > data.speed * 3.6 * del_time * 2: # 距离阈值
continue
elif data.speed == last_point.speed and data.speed > 0 and data.direction == last_point.direction:
# 非精确
continue
else:
data.dist = dist
# del_list.append(del_time)
new_trace.append(data)
else:
data.dist = 0
new_trace.append(data)
last_point = cur_point
# gps_point, gps_mean, gps_med = exclude_abnormal(del_list)
return new_trace
def print_data(trace, bi, ei):
fp = open('./data/1.txt', 'w')
lx, ly = -1, -1
for data in trace[bi:ei + 1]:
b, l = xy2bl(data.px, data.py)
if lx != -1:
dist = calc_dist([lx, ly], [data.px, data.py])
else:
dist = 10.01
if dist > 10:
fp.write("{0},{1},{2},{3}\n".format(data.px, data.py, l, b))
lx, ly = data.px, data.py
# print data.state, data.car_state, data.speed, data.stime, data.px, data.py, data.dist
fp.close()
def get_trace_dyn(last_edge, cur_edge, last_point, cur_point, cnt=-1):
"""
通过路网计算两点之间的路径
:param last_edge: 上一条匹配边
:param cur_edge: 当前匹配的边
:param last_point: 上一个GPS点
:param cur_point: 当前GPS点
:return: trace, dist of trace
"""
spoint, _, _ = point_project_edge(last_point, last_edge)
try:
epoint, _, _ = point_project_edge(cur_point, cur_edge)
except TypeError:
print 'get_trace_dyn', cnt
if last_edge == cur_edge:
return [spoint, epoint], calc_dist(spoint, epoint)
node_set = set()
q = Queue.PriorityQueue(maxsize=-1)
init_candidate_queue(last_point, last_edge, q, node_set)
# 使用Dijkstra算法搜索路径
edge_found = False
while not q.empty() and not edge_found:
dnode = q.get()
cur_node, cur_dist = dnode.node, dnode.dist
for edge, node in cur_node.link_list:
if node.nodeid in node_set:
continue
node_set.add(node.nodeid)
next_dnode = DistNode(node, cur_dist + edge.edge_length)
node.prev_node = cur_node
q.put(next_dnode)
if edge.edge_index == cur_edge.edge_index:
edge_found = True
trace = [cur_point]
n0, n1 = cur_edge.node0, cur_edge.node1
if n0.prev_node == n1: # n1 is nearer from last point
cur_node = n1
else:
cur_node = n0
while cur_node != last_edge.node0 and cur_node != last_edge.node1:
trace.append(cur_node.point)
cur_node = cur_node.prev_node
trace.append(cur_node.point)
trace.append(spoint)
return trace, get_trace_dist(trace)
def add_pt(last_pt, pt):
pt_list = []
v0 = np.array(last_pt)
v1 = np.array(pt)
dv = v1 - v0
dist = calc_dist(pt, last_pt)
uv = dv / dist
cv = v0
while dist > DIST_THREAD * 2:
v = cv + uv * DIST_THREAD
dist -= DIST_THREAD
pt_list.append([v[0], v[1]])
cv = v
v = cv + uv * dist / 2
pt_list.append([v[0], v[1]])
return pt_list
def process_road_list(self, road_list):
"""
处理road list，放入类中的基本数据结构中
:param road_list: [Road]
:return:
"""
map_temp_node = {} # 维护地图节点
road_point = {} # temp dict, save road points list
for road in road_list:
rid = road.rid
r = MapRoad(road.name, road.ort, 1)
self.map_road[rid] = r
pt_list = []
for point in road.point_list:
pt_list.append([point.px, point.py])
road_point[rid] = [pt_list, road.ort]
# 维护节点dict
# 每一个点都单独列入dict，边用两点表示，道路用若干点表示
for rid, road_info in road_point.iteritems():
r = self.map_road[rid]
last_nodeid = -1
lastx, lasty = None, None
xylist, ort = road_info[:]
for i, xy in enumerate(xylist):
x, y = round(xy[0], 2), round(xy[1], 2)
str_bl = "{0},{1}".format(x, y)
# x, y = bl2xy(lat, lng)
try:
nodeid = map_temp_node[str_bl]
nd = self.map_node[nodeid]
except KeyError:
nodeid = len(map_temp_node)
map_temp_node[str_bl] = nodeid
nd = MapNode([x, y], nodeid)
self.map_node[nodeid] = nd
if i > 0:
edge_length = calc_dist([x, y], [lastx, lasty])
oneway = False if ort == 3 else True
edge = MapEdge(self.map_node[last_nodeid], self.map_node[nodeid], oneway, len(self.map_edge),
edge_length, rid)
self.map_edge.append(edge)
r.add_node(nd)
last_nodeid, lastx, lasty = nodeid, x, y
def load_trace():
fp = open('./data/1.txt')
lx, ly, dist = -1, -1, 0
trace_list = []
for line in fp.readlines():
item = line.strip('\n').split(',')
px, py = map(float, item[0:2])
lng, lat = map(float, item[2:4])
lng, lat = round(lng, 6), round(lat, 6)
if lx != -1:
d = geo.calc_dist([px, py], [lx, ly])
dist += d
lx, ly = px, py
trace_list.append([lng, lat])
split_len = int(math.ceil(len(trace_list) / 16.0))
t_list = [trace_list[0]]
sp_list = trace_list[1:-1:split_len]
t_list.extend(sp_list)
t_list.append(trace_list[-1])
return trace_list, dist
def split_road():
conn1 = sqlite3.connect("../map_info/hz1.db")
cursor = conn1.cursor()
sql = "select * from TB_SEG_POINT order by s_id, seq;"
cursor.execute(sql)
roads = defaultdict(list)
for item in cursor:
s_id, seq, lng, lat = item[1:]
x, y = bl2xy(lat, lng)
roads[s_id].append([x, y])
pt_tup_list = []
for sid, pt_list in roads.iteritems():
last_pt = None
new_pt_list = []
for pt in pt_list:
if last_pt is not None:
dist = calc_dist(pt, last_pt)
if dist > DIST_THREAD:
for n_pt in add_pt(last_pt, pt):
new_pt_list.append(n_pt)
last_pt = pt
new_pt_list.append(pt)
roads[sid] = new_pt_list
for i, pt in enumerate(new_pt_list):
b, l = xy2bl(pt[0], pt[1])
t = (sid, i + 1, l, b)
pt_tup_list.append(t)
cursor.close()
conn1.close()
conn2 = sqlite3.connect("../map_info/hz2.db")
cursor = conn2.cursor()
sql2 = "insert into tb_seg_point(point_id, s_id, seq, longi, lati) values(0,?,?,?,?)"
cursor.executemany(sql2, pt_tup_list)
conn2.commit()
cursor.close()
def get_all_gps_data():
"""
从数据库中获取一段时间的GPS数据
:return:
"""
end_time = datetime(2018, 5, 8, 6, 0, 0)
conn = cx_Oracle.connect('hz/[email protected]:1521/orcl')
bt = clock()
begin_time = end_time + timedelta(minutes=-5)
# sql = "select px, py, speed_time, state, speed, carstate, direction, vehicle_num from " \
# "TB_GPS_1805 t where speed_time >= :1 " \
# "and speed_time < :2 and vehicle_num = '浙AT7902' order by speed_time "
sql = "select px, py, speed_time, state, speed, carstate, direction, vehicle_num from " \
"TB_GPS_1805 t where speed_time >= :1 and speed_time < :2"
tup = (begin_time, end_time)
cursor = conn.cursor()
cursor.execute(sql, tup)
veh_trace = {}
static_num = {}
for item in cursor.fetchall():
lng, lat = map(float, item[0:2])
if 119 < lng < 121 and 29 < lat < 31:
px, py = bl2xy(lat, lng)
state = int(item[3])
stime = item[2]
speed = float(item[4])
car_state = int(item[5])
ort = float(item[6])
veh = item[7][-6:]
# if veh != 'AT0956':
# continue
taxi_data = TaxiData(veh, px, py, stime, state, speed, car_state,
ort)
try:
veh_trace[veh].append(taxi_data)
except KeyError:
veh_trace[veh] = [taxi_data]
try:
static_num[veh] += 1
except KeyError:
static_num[veh] = 1
et = clock()
new_dict = {}
for veh, trace in veh_trace.iteritems():
trace.sort(cmp1)
new_trace = []
last_data = None
i = 0
for data in trace:
esti = True
dist = 0
if last_data is not None:
dist = calc_dist([data.px, data.py],
[last_data.px, last_data.py])
dt = (data.stime - last_data.stime).total_seconds()
if data.state == 0:
esti = False
# 过滤异常
if dt <= 10 or dt > 180:
esti = False
elif dist > 100 / 3.6 * dt: # 距离阈值
esti = False
elif data.car_state == 1: # 非精确
esti = False
elif data.speed == last_data.speed and data.direction == last_data.direction:
esti = False
elif dist < 20: # GPS的误差在10米，不准确
esti = False
last_data = data
if esti:
new_trace.append(data)
# print i, dist
# i += 1
new_dict[veh] = new_trace
print "get all gps data", et - bt
# print "all car:{0}, ave:{1}".format(len(static_num), len(trace) / len(static_num))
cursor.close()
conn.close()
return new_dict
def get_gps_data_from_redis():
bt = clock()
conn = redis.Redis(host="192.168.11.229", port=6300, db=1)
keys = conn.keys()
new_trace = {}
if len(keys) != 0:
m_res = conn.mget(keys)
et = clock()
veh_trace = {}
static_num = {}
for data in m_res:
try:
js_data = json.loads(data)
lng, lat = js_data['longi'], js_data['lati']
veh, str_time = js_data['isu'], js_data['speed_time']
speed = js_data['speed']
stime = datetime.strptime(str_time, "%Y-%m-%d %H:%M:%S")
state = 1
car_state = 0
ort = js_data['ort']
if 119 < lng < 121 and 29 < lat < 31:
px, py = bl2xy(lat, lng)
taxi_data = TaxiData(veh, px, py, stime, state, speed,
car_state, ort)
try:
veh_trace[veh].append(taxi_data)
except KeyError:
veh_trace[veh] = [taxi_data]
try:
static_num[veh] += 1
except KeyError:
static_num[veh] = 1
except TypeError:
pass
print "redis cost ", et - bt
cnt = {}
for veh, trace in veh_trace.iteritems():
try:
cnt[len(trace)] += 1
except KeyError:
cnt[len(trace)] = 1
print cnt
for veh, trace in veh_trace.iteritems():
trace.sort(cmp1)
filter_trace = []
last_data = None
for data in trace:
esti = True
if last_data is not None:
dist = calc_dist([data.px, data.py],
[last_data.px, last_data.py])
dt = (data.stime - last_data.stime).total_seconds()
if data.state == 0:
esti = False
# 过滤异常
if dt <= 10 or dt > 180:
esti = False
elif dist > 100 / 3.6 * dt: # 距离阈值
esti = False
elif data.car_state == 1: # 非精确
esti = False
# elif data.speed == last_data.speed and data.direction == last_data.direction:
# esti = False
elif dist < 10: # GPS的误差在10米，不准确
esti = False
last_data = data
if esti:
filter_trace.append(data)
if 4 <= len(filter_trace) <= 40:
new_trace[veh] = filter_trace
# cnt static
cnt = {}
for veh, trace in new_trace.iteritems():
try:
cnt[len(trace)] += 1
except KeyError:
cnt[len(trace)] = 1
# if len(trace) == 13:
# print veh
# print "get all gps data {0}".format(len(veh_trace))
# print "all car:{0}, ave:{1}".format(len(static_num), len(trace) / len(static_num))
return new_trace
def get_all_gps_data(end_time):
"""
从数据库中获取一段时间的GPS数据
:param end_time
:return:
"""
# end_time = datetime(2018, 5, 1, 5, 0, 0)
conn = cx_Oracle.connect('hz/[email protected]:1521/orcl')
begin_time = end_time + timedelta(hours=-3)
# sql = "select px, py, speed_time, state, speed, carstate, direction, vehicle_num from " \
# "TB_GPS_1805 t where speed_time >= :1 " \
# "and speed_time < :2 and vehicle_num = '浙AT7484' order by speed_time "
sql = "select px, py, speed_time, state, speed, carstate, direction, vehicle_num from " \
"TB_GPS_1805 t where speed_time >= :1 and speed_time < :2"
tup = (begin_time, end_time)
cursor = conn.cursor()
cursor.execute(sql, tup)
veh_trace = {}
static_num = {}
for item in cursor.fetchall():
lng, lat = map(float, item[0:2])
if 119 < lng < 121 and 29 < lat < 31:
px, py = bl2xy(lat, lng)
state = int(item[3])
stime = item[2]
speed = float(item[4])
car_state = int(item[5])
ort = float(item[6])
veh = item[7][-6:]
# if veh != 'AT0956':
# continue
taxi_data = TaxiData(veh, px, py, stime, state, speed, car_state,
ort)
try:
veh_trace[veh].append(taxi_data)
except KeyError:
veh_trace[veh] = [taxi_data]
try:
static_num[veh] += 1
except KeyError:
static_num[veh] = 1
new_dict = {}
record_cnt = 0
for veh, trace in veh_trace.iteritems():
trace.sort(cmp1)
new_trace = []
last_data = None
for data in trace:
esti = True
if last_data is not None:
dist = calc_dist([data.px, data.py],
[last_data.px, last_data.py])
dt = (data.stime - last_data.stime).total_seconds()
if data.state == 0:
esti = esti
# 过滤异常
if dt <= 10:
esti = False
elif dist > 100 / 3.6 * dt: # 距离阈值
esti = False
elif data.car_state == 1: # 非精确
esti = False
elif data.speed == last_data.speed and data.direction == last_data.direction:
esti = False
elif dist < 15: # GPS的误差在10米，不准确
esti = False
last_data = data
if esti:
new_trace.append(data)
# print i, dist
# i += 1
if len(new_trace) >= 5:
new_dict[veh] = new_trace
record_cnt += len(new_trace)
# print "all car records", record_cnt, "car number", len(new_dict)
cursor.close()
conn.close()
# check_itv(new_dict)
return new_dict
def DYN_MATCH(traj_order):
"""
T.B.M.
using point match with dynamic programming,
:param traj_order: list of Taxi_Data
:return:
"""
kdt, X = make_kdtree()
first_point = True
last_data, last_point = None, None
last_state = 0 # 判断双向道路当前是正向或者反向
total_dist = 0.0 # 计算路程
last_time = None
cnt = 0
match_results = [] # MatchResult
for data in traj_order:
bt = clock()
if first_point:
# 第一个点
candidate_edges = get_candidate_first(data, kdt, X)
# Taxi_Data .px .py .stime .speed
first_point = False
mod_point, last_edge, dist = get_mod_point(data, candidate_edges,
last_point, cnt)
mr = MatchResult([data.px, data.py])
mr.add_match(last_edge.edge_index, mod_point, [], dist, dist)
mr.set_first(True) # 首次匹配
match_results.append(mr)
last_point = mod_point
else:
# 随后的点
# 首先判断两个点是否离得足够远
T = 10
cur_point = [data.px, data.py]
interval = calc_dist(cur_point, last_point)
# print cnt, interval
if interval < T:
last_time = data.stime
continue
cur_point = [data.px, data.py]
interval_time = (data.stime - last_time).total_seconds()
last_mr = match_results[cnt - 1]
has_result = False
ret = {}
# 对于上一次匹配的每一条记录进行运算
mr = MatchResult(cur_point)
for i, mp in enumerate(last_mr.match_point_list):
last_index = mp.edge_index
last_edge = map_edge_list[last_index]
candidate_edges = get_candidate_later(data, last_data,
last_point, last_edge,
last_state,
interval_time, cnt)
# if cnt == 9:
# draw_edge_list(candidate_edges)
if len(candidate_edges) > 0:
# 正常情形
match_list = get_mod_points(data, candidate_edges,
last_point, last_edge, cnt)
for mtc in match_list:
mp, edge, dist, score = mtc[0:4]
ei = edge.edge_index
try:
ret[ei][1].append(last_index)
except KeyError:
ret[ei] = [mp, [last_index], dist, score]
for ei, v in ret.iteritems():
has_result = True
mr.add_match(ei, ret[ei][0], ret[ei][1], ret[ei][2],
ret[ei][3])
if has_result is False: # 无匹配结果
candidate_edges = get_candidate_first(data, kdt, X)
# Taxi_Data .px .py .stime .speed
first_point = False
mod_point, last_edge, dist = get_mod_point(
data, candidate_edges, last_point, cnt)
mr.add_match(last_edge.edge_index, mod_point, [], dist, dist)
mr.set_first(not has_result)
match_results.append(mr)
# print len(mr.match_list)
last_point = cur_point
plt.text(data.px, data.py, '{0}'.format(cnt))
str_time = data.stime.strftime('%M:%S')
# plt.text(data.px, data.py, '{0},{1},{2}'.format(cnt, str_time, data.speed))
# print cnt, clock() - bt
cnt += 1
last_time = data.stime
last_data = data
return match_results, total_dist
def PNT_MATCH(traj_order):
"""
using point match with topology,
:param traj_order: list of Taxi_Data
:return:
"""
kdt, X = make_kdtree()
first_point = True
last_data, last_point, last_edge = None, None, None
last_state = 0 # 判断双向道路当前是正向或者反向
total_dist = 0.0 # 计算路程
last_time = None
cnt = 0
traj_mod = []
for data in traj_order:
if first_point:
# 第一个点
candidate_edges = get_candidate_first(data, kdt, X)
# Taxi_Data .px .py .stime .speed
first_point = False
mod_point, last_edge, _ = get_mod_point(data, candidate_edges,
last_point, cnt)
state = 'c'
traj_mod.append(mod_point)
last_point = mod_point
last_time = data.stime
else:
# 随后的点
# 首先判断两个点是否离得足够远
T = 15
cur_point = [data.px, data.py]
interval = calc_dist(cur_point, last_point)
interval_time = (data.stime - last_time).total_seconds()
# print cnt, interval
if interval < T:
last_time = data.stime
continue
candidate_edges = get_candidate_later(data, last_data, last_point,
last_edge, last_state,
interval_time, cnt)
# if cnt == 23:
# print data.stime, last_time, interval_time
# draw_edge_list(candidate_edges)
if len(candidate_edges) == 0:
# no match, restart
candidate_edges = get_candidate_first(data, kdt, X)
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges,
None, cnt)
state = 'c'
else:
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges,
last_point, cnt)
state = 'r'
offset_dist = calc_dist(mod_point, cur_point)
if offset_dist > 60:
# 判断是否出现太远的情况
candidate_edges = get_candidate_first(data, cnt, X)
# draw_edge_list(candidate_edges)
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges,
None, cnt)
state = 'm'
if state == 'r':
trace = get_trace(last_edge, cur_edge, last_point, mod_point)
draw_seg(trace, 'b')
dist = get_trace_dist(trace)
else:
dist = calc_dist(cur_point, last_point)
total_dist += dist
traj_mod.append(mod_point)
last_point, last_edge = cur_point, cur_edge
plt.text(data.px, data.py, '{0}'.format(cnt))
plt.text(mod_point[0], mod_point[1], '{0}'.format(cnt), color=state)
cnt += 1
last_time, last_data = data.stime, data
# print cnt, data.px, data.py, mod_point[0], mod_point[1]
fp.close()
return traj_mod, total_dist
def center_2():
"""
修正曲折道路
最后再去除重复点
:return:
"""
road_data = load_model('./road_new/center1.txt')
# mod_road = {54}
new_road_data = []
for road in road_data:
polyline = road['polyline']
name = road['name']
# rid = int(road['rid'])
xy_list = polyline2xylist(polyline)
new_xy_list = []
# 先找出最接近的那个点
last_xy = xy_list[0]
new_xy_list.append(last_xy)
del xy_list[0]
len_list = len(xy_list)
for i in range(len_list):
min_dist, sel = 1e10, -1
for j, xy in enumerate(xy_list):
dist = calc_dist(last_xy, xy)
if dist < min_dist:
min_dist, sel = dist, j
last_xy = xy_list[sel]
new_xy_list.append(last_xy)
del xy_list[sel]
new_xy_list = reversed(new_xy_list)
# 去重
temp_list = []
last_xy = [0, 0]
for x, y in new_xy_list:
if x == last_xy[0] and y == last_xy[1]:
print 'duplicated', name
continue
temp_list.append([x, y])
last_xy = [x, y]
new_xy_list = temp_list
xy_list = new_xy_list[:]
new_xy_list = []
# 再来一遍
last_xy = xy_list[0]
new_xy_list.append(last_xy)
del xy_list[0]
len_list = len(xy_list)
for i in range(len_list):
min_dist, sel = 1e10, -1
for j, xy in enumerate(xy_list):
dist = calc_dist(last_xy, xy)
if dist < min_dist:
min_dist, sel = dist, j
last_xy = xy_list[sel]
new_xy_list.append(last_xy)
del xy_list[sel]
new_xy_list = reversed(new_xy_list)
# 去重
temp_list = []
last_xy = [0, 0]
for x, y in new_xy_list:
if x == last_xy[0] and y == last_xy[1]:
print 'duplicated', name
continue
temp_list.append([x, y])
last_xy = [x, y]
new_xy_list = temp_list
road['polyline'] = xylist2polyline(new_xy_list)
new_road_data.append(road)
save_model('./road_new/center1.txt', new_road_data)
def in_area(pt, area_pt):
return calc_dist(pt, area_pt) < 200
def gene_from_gps_data():
"""
从gps点记录里面获取轨迹，并生成出发点和到达点
:return:
"""
try:
gps_data = {}
# gps_data 用于存放每辆车的轨迹
fp = open('./road/road.txt', 'w')
conn = cx_Oracle.connect('hz', 'hz', '192.168.11.88/orcl')
sql = "select vehicle_num, px, py from tb_gps_1805 where speed_time > to_date(" \
"'2018-05-01 9:00:00', 'yyyy-mm-dd hh24:mi:ss')" \
"and speed_time < to_date('2018-05-01 10:00:00', 'yyyy-mm-dd hh24:mi:ss')"
cursor = conn.cursor()
bt = time.clock()
cursor.execute(sql)
cnt = 0
for item in cursor:
data = map(float, item[1:3])
lng, lat = data[0:2]
if lng > 121 or lng < 119 or lat > 31 or lat < 29:
continue
x, y = bl2xy(lat, lng)
veh = item[0][-6:]
try:
gps_data[veh].append([x, y])
except KeyError:
gps_data[veh] = [[x, y]]
cnt += 1
conn.close()
et = time.clock()
print et - bt
s = 0
req_list = []
for veh, data_list in gps_data.iteritems():
s += len(data_list)
use_list = data_list[::10]
for i in range(len(use_list) - 1):
req_list.append([data_list[i][0], data_list[i][1], data_list[i + 1][0], data_list[i + 1][1]])
print s
pa_list, cnt, idx = [], 0, 0
for req in req_list:
x0, y0, x1, y1 = req[0:4]
if calc_dist([x0, y0], [x1, y1]) > 2000:
b0, l0 = xy2bl(x0, y0)
b1, l1 = xy2bl(x1, y1)
pa_list.append([l0, b0, l1, b1])
idx += 1
if idx >= 20:
batch_path_fetch(pa_list, fp)
time.sleep(0.2)
pa_list = []
idx = 0
cnt += 1
print "batch" + str(cnt)
if cnt >= 1000:
break
fp.close()
except Exception as e:
print e.message
print("Exitiing application.")
exit()
lat_deg = cust_rec_json['latitude']
if is_prop_latlon(lat_deg, True):
lat = deg2rad(float(lat_deg))
else:
print("Customer {}'s latitude is not in a proper float format.".
format(user_id))
print("Exitiing application.")
exit()
lon_deg = cust_rec_json['longitude']
if is_prop_latlon(lon_deg, False):
lon = deg2rad(float(lon_deg))
else:
print("Customer {}'s longitude is not in a proper float format.".
format(user_id))
print("Exitiing application.")
exit()
dist = calc_dist(float(config['DEFAULT']['EARTH_RAD']), off_lat,
off_lon, lat, lon)
if (dist < 100):
mat_cust_dict[user_id] = \
cust_rec_json['name']
# Sort the customer records wrt their IDs and print...
for user_id in sorted(iterkeys(mat_cust_dict)):
print('{}: {}'.format(user_id, mat_cust_dict[user_id]))
def POINT_MATCH(traj_order):
"""
using point match with topology,
:param traj_order: list of Taxi_Data
:return:
"""
first_point = True
last_point, last_edge = None, None
last_state = 0 # 判断双向道路当前是正向或者反向
cnt = 0
traj_mod = []
for data in traj_order:
if first_point:
# 第一个点
candidate_edges = get_candidate_first(data, cnt)
# Taxi_Data .px .py .stime .speed
first_point = False
mod_point, last_edge, _ = get_mod_point(data, candidate_edges, last_point, cnt)
state = 'c'
traj_mod.append(mod_point)
last_point = mod_point
else:
# 随后的点
# 首先判断两个点是否离得足够远
T = 15
cur_point = [data.px, data.py]
interval = calc_dist(cur_point, last_point)
# print cnt, interval
if interval < T:
continue
candidate_edges = get_candidate_later(cur_point, last_point, last_edge, last_state, cnt)
if len(candidate_edges) == 0:
# no match, restart
candidate_edges = get_candidate_first(data, cnt)
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges, None, cnt)
state = 'c'
else:
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges, last_point, cnt)
state = 'r'
if state == 'r':
trace = get_trace(last_edge, cur_edge, last_point, mod_point)
draw_seg(trace, 'b')
offset_dist = calc_dist(mod_point, cur_point)
if offset_dist > 100:
# 判断是否出现太远的情况
candidate_edges = get_candidate_first(data, cnt)
# draw_edge_list(candidate_edges)
mod_point, cur_edge, _ = get_mod_point(data, candidate_edges, None, cnt)
state = 'm'
traj_mod.append(mod_point)
last_point, last_edge = cur_point, cur_edge
plt.text(data.px, data.py, '{0}'.format(cnt))
plt.text(mod_point[0], mod_point[1], '{0}'.format(cnt), color=state)
cnt += 1
# print cnt, data.px, data.py, mod_point[0], mod_point[1]
if 77 <= cnt <= 127:
result = '{0},{1}\n'.format(mod_point[0], mod_point[1])
fp.write(result)
fp.close()
return traj_mod
def DYN_MATCH(traj_order):
"""
using point match with dynamic programming,
:param traj_order: list of Taxi_Data
:return:
"""
first_point = True
last_point, last_edge = None, None
last_state = 0 # 判断双向道路当前是正向或者反向
cnt = 0
traj_mod = [] # 存放修正偏移后的data
for data in traj_order:
if first_point:
candidate_edges = get_candidate_first(data, cnt)
# Taxi_Data .px .py .stime .speed
first_point = False
mod_point, last_edge, score = get_mod_point(data, candidate_edges, last_point, cnt)
state = 'c'
data.set_edge([last_edge, score])
traj_mod.append(data)
last_point = mod_point
else:
# 首先判断两个点是否离得足够远
T = 10000 / 3600 * 10
cur_point = [data.px, data.py]
interval = calc_dist(cur_point, last_point)
# print cnt, interval
if interval < T:
continue
# 读取上一个匹配点的信息
last_data = traj_mod[cnt - 1]
last_point = [last_data.px, last_point.py]
min_score, sel_edge, sel_score = 1e10, None, 0
for last_edge, last_score in last_data.edge_info:
candidate_edges = get_candidate_later(cur_point, last_point, last_edge, last_state, cnt)
if len(candidate_edges) == 0:
# no match, restart
candidate_edges = get_candidate_first(data, cnt)
mod_point, cur_edge, score = get_mod_point(data, candidate_edges, None, cnt)
state = 'c'
cur_score = score + 1e5
else:
# if cnt == 147:
# draw_edge_list(candidate_edges)
mod_point, cur_edge, score = get_mod_point(data, candidate_edges, last_point, cnt)
cur_score = score + last_score
state = 'r'
if cur_score < min_score:
min_score, sel_edge, sel_score = cur_score, cur_edge, score
# if state == 'r':
# trace = get_trace(last_edge, cur_edge, last_point, mod_point)
# draw_seg(trace, 'b')
offset_dist = calc_dist(mod_point, cur_point)
if offset_dist > 100:
# 判断是否出现太远的情况
candidate_edges = get_candidate_first(data, cnt)
# draw_edge_list(candidate_edges)
mod_point, cur_edge = get_mod_point(data, candidate_edges, None, cnt)
state = 'm'
traj_mod.append(data)
last_point, last_edge = cur_point, cur_edge
plt.text(data.px, data.py, '{0}'.format(cnt))
plt.text(mod_point[0], mod_point[1], '{0}'.format(cnt), color=state)
cnt += 1
print cnt, data.px, data.py
return traj_mod
def estimate_road_speed(last_edge,
cur_edge,
last_point,
cur_point,
last_data,
cur_data,
cnt=-1):
"""
通过路网估计两条数据之间各个路段的速度
:param last_edge: 上一条匹配边 MapEdge
:param cur_edge: 当前匹配的边
:param last_point: 上一个匹配到的GPS点 [x, y]
:param cur_point: 当前匹配到的GPS点
:param last_data: 上一次记录的GPS数据 TaxiData
:param cur_data: 本次记录的GPS数据
:param cnt: for debug int
:return: trace, travel list : travel data:[edge(list of [x, y]), edge_speed, direction]
"""
# print cnt
spoint, epoint = last_point, cur_point
# 首先是同一条边的情况
if last_edge == cur_edge:
cur_dist = calc_dist(spoint, epoint)
itv_time = (cur_data.stime - last_data.stime).total_seconds()
observed_spd = cur_dist / itv_time * 3.6
# 判断方向
angle = calc_included_angle_math(spoint, epoint, cur_edge.node0.point,
cur_edge.node1.point)
match_direction = True if angle > 0 else False
# print cur_edge.way_id, match_direction
return [spoint, epoint], [[cur_edge, observed_spd, match_direction]]
# 使用Dijkstra算法搜索路径
# 加入优先队列(最小堆)优化速度
# 基准测试平均每次执行速度在万分之一秒左右 0.0001s，主要因为道路拓扑不是很复杂
# 理论上应该用A*的H来做优先队列value函数
# 并将H超过cur_dist的剪枝，不过在道路拓扑并不复杂的情况下，这个剪枝效率能提升多少存疑
node_set = set()
q = Queue.PriorityQueue(maxsize=-1)
init_candidate_queue(last_point, last_edge, q, node_set)
esti_cnt = 0
dist_thread = calc_dist(spoint, epoint) * 2.0
edge_found = False
while not q.empty() and not edge_found:
dnode = q.get()
cur_node, cur_dist = dnode.node, dnode.dist
if cur_dist > dist_thread:
break
# 宽度优先
for edge, node in cur_node.link_list:
esti_cnt += 1
if node.nodeid in node_set:
continue
if edge.node0 == node:
edge.match_direction = False
else:
edge.match_direction = True
node_set.add(node.nodeid)
next_dnode = DistNode(node, cur_dist + edge.edge_length)
# 在每个MapNode里面记录下搜索路径
node.prev_node, node.prev_edge = cur_node, edge
q.put(next_dnode)
if edge.edge_index == cur_edge.edge_index:
edge_found = True
# print "esti", esti_cnt
if not edge_found:
return [], []
# 根据纪录重建路径
# 终点
trace = [cur_point]
n0, n1 = cur_edge.node0, cur_edge.node1
if n0.prev_node is None or n0.prev_node == n1: # n1 is nearer from last point
cur_node = n1
else:
cur_node = n0
dist = calc_dist(cur_point, cur_node.point)
# Here, Add edge
travel = [] # travel edge list, from last edge to current edge
if dist > 0:
travel.append([cur_edge, dist, cur_edge.match_direction])
# 逆推
while cur_node != last_edge.node0 and cur_node != last_edge.node1:
trace.append(cur_node.point)
prev_edge = cur_node.prev_edge
travel.append(
[prev_edge, prev_edge.edge_length, prev_edge.match_direction])
cur_node = cur_node.prev_node
# 起点
dist = calc_dist(last_point, cur_node.point)
match_direction = True if cur_node == last_edge.node1 else False
if dist > 0:
travel.append([last_edge, dist, match_direction])
# 所以要倒序
travel = travel[::-1]
travel_dist = 0.0
# 简单加一下
for edge, dist, _ in travel:
travel_dist += dist
itv_time = (cur_data.stime - last_data.stime).total_seconds()
# 得到如下值
# 1.观测得到的行驶距离，行驶时间，可以得到平均速度
observed_spd = travel_dist / itv_time * 3.6
# 2.之前一个点的速度 当前点的速度
last_spd, cur_spd = last_data.speed, cur_data.speed
# 3.行驶各路段
trace.append(cur_node.point)
trace.append(spoint)
# print "estimate,{0} ".format(cnt), et - bt
spd_list = get_speed_list_a1(travel, last_spd, cur_spd, observed_spd,
itv_time)
# for edge, spd, ort in spd_list:
# print edge.way_id, ort
return trace, spd_list
