def get_update_time_interval():
#打开xml文档
dom = xml.dom.minidom.parse(get_current_dir() + '/properties.xml')
#得到文档元素对象
root = dom.documentElement
update_time_interval = root.getElementsByTagName('update_time_interval')
#获得更新时间间隔
update_time = int(update_time_interval[0].firstChild.data)
return dict({'update_time': update_time})
def get_username_and_password():
#打开xml文档
dom = xml.dom.minidom.parse(get_current_dir() + '/properties.xml')
#得到文档元素对象
root = dom.documentElement
mysql_uesrname = root.getElementsByTagName('mysql_uesrname')
mysql_password = root.getElementsByTagName('mysql_password')
#获得用户名与密码
user = str(mysql_uesrname[0].firstChild.data)
passwd = str(mysql_password[0].firstChild.data)
if passwd == 'null': passwd = ''
return dict({'user': user, 'passwd': passwd})
print filepath + " removed!"
#如果是文件夹则整个文件夹删除
elif os.path.isdir(filepath):
shutil.rmtree(filepath, True)
print "dir " + filepath + " removed!"
#清空mongodb数据库
def clearmongodb():
client = Client()
db = client.traffic_project
#清空速度
db.speed.remove({})
#清空状态
db.state.remove({})
#清空流量
db.maccount.remove({})
#清空清空拥堵指数
db.congestion_index.remove({})
#清空其他指标数据
db.traffic_state.remove({})
#主入口
#清楚各路段的旧记录
seglist = getsegmentlist()
for seg in seglist:
filename = get_current_dir() + "/data/wifiresult/" + str(seg['segmentid'])
removedir(filename)
clearmongodb()
def fatherthread():
#测试数据开始时间
test_begin_time = 1477843200
#测试数据总时长
test_time_long = 24
#模拟测试数据单前时间
delta = int(time.time()) - test_begin_time
current_time = int(time.time()) - delta
#当前日期，格式yyyymmdd
date = timestamp2date(current_time)
#探针数据目录
sourcefolder = get_current_dir() + "/WiFiData/"
#数据处理结果存放目录
resultfolder = get_current_dir() + "/data/wifiresult/"
print "程序启动日期：" + date
print "程序启动时间：" + timestamp2str(current_time)
#循环执行计算任务，每5分钟计算一次，知道指定时间停止
while current_time <= (test_begin_time + test_time_long * 3600):
print "当前时间：" + timestamp2str(current_time)
timestr = time.strftime('%Y%m%d%H%M%S', time.localtime(current_time))
speed_d = {'time': timestr}
mac_d = {'time': timestr}
state_d = {'time': timestr}
congestion_index_d = {'time': timestr}
#获取历史速度数据(不含当前)
his_df = gethistory(current_time, resultfolder)
#耗时计算的起始时刻
start_time = time.time()
#任务分块,确定各个子线程的输入参数
inputs = []
#从数据库获得路段列表
segmentlist = getsegmentlist()
#任务切割与分配
block_q = separation(len(segmentlist), 90)
while not block_q.empty():
block = block_q.get()
start = block.get('start')
end = block.get('end')
inputs.append({
'seglist': segmentlist[start:end + 1],
'date': date,
'current_time': current_time,
'sourcefolder': sourcefolder,
'resultfolder': resultfolder,
'history': his_df
})
#使用multiprocessing并行进程池来并行处理任务，充分利用cup资源
#确定进程池大小
pool_size = multiprocessing.cpu_count() * 2
print "进程数" + str(pool_size)
# 初始化线程池
pool = multiprocessing.Pool(processes=pool_size,
initializer=start_process,
maxtasksperchild=2)
# 获得各子线程的返回结果，存在列表final_result里
final_result = pool.map(
sonthread,
inputs,
)
#关闭进程池，阻止其他进程进入，等待各个任务完成
pool.close()
#将进程池join到父线程里
pool.join()
pool.terminate()
#父线程在所有子线程结束后，把结果取出来整合
for list in final_result:
mac_d = dict(mac_d, **list[0])
speed_d = dict(speed_d, **list[1])
state_d = dict(state_d, **list[2])
congestion_index_d = dict(congestion_index_d, **list[3])
# mac_d=sorted(mac_d.iteritems(),key=lambda d:d[0])
# speed_d=sorted(speed_d.iteritems(),key=lambda d:d[0])
# state_d=sorted(state_d.iteritems(),key=lambda d:d[0])
# congestion_index_d=sorted(congestion_index_d.iteritems(),key=lambda d:d[0])
#插入速度，流量，拥堵状态数据
print speed_d
print mac_d
print state_d
print congestion_index_d
#车流量
insert_maccount(mac_d)
#速度
insert_speed(speed_d)
#状态
insert_state(state_d)
#拥堵指数
insert_congestion_index(congestion_index_d)
#交通指数
traffic_index = get_traffic_index(timestr)
#路网拥堵比率
congrestion_rate = get_congrestion_rate(timestr)
traffic_state_dict = {
'time': timestr,
'traffic_index': traffic_index,
'congrestion_rate': congrestion_rate
}
#区域交通指数
area_name = getAreas()
for area in area_name:
area_dict = {area: get_traffic_index_of_area(timestr, area)}
traffic_state_dict = dict(traffic_state_dict, **area_dict)
print traffic_state_dict
#交通状态指标插入数据库
insert_traffic_state(traffic_state_dict)
#标记耗时
time_cost = time.time() - start_time
print "用时" + str(time_cost)
#线程休眠，
# 休眠间隔与更新周期相关
sleeptime = get_update_time_interval()['update_time'] - time_cost
if sleeptime < 0:
sleeptime = 0
time.sleep(sleeptime)
#更新当前时间
current_time = current_time + get_update_time_interval()['update_time']
def fatherthread():
current_time = int(time.time())
#current_time=1492297200
#当前日期，格式yyyymmdd
date = timestamp2date(current_time)
#探针数据目录
sourcefolder = "/home/wicloud/ywbrtdata/"
#sourcefolder="/mnt/home/zhangyunlong/ywbrt/ywbrtdata/"
#数据处理结果存放目录
resultfolder = get_current_dir() + "/data/wifiresult/"
print "程序启动日期：" + date
print "程序启动时间：" + timestamp2str(current_time)
#循环执行计算任务，每5分钟计算一次
#更新周期可以在配置文件中设定
#该程序是用于真正的实时数据处理的，作为一个服务一直执行
while True:
print "当前时间：" + timestamp2str(current_time)
timestr = time.strftime('%Y%m%d%H%M%S', time.localtime(current_time))
speed_d = {'time': timestr}
mac_d = {'time': timestr}
state_d = {'time': timestr}
congestion_index_d = {'time': timestr}
#获取历史速度数据(不含当前)
his_df = gethistory(current_time, resultfolder)
#耗时计算的起始时刻
start_time = time.time()
#任务分块,确定各个子线程的输入参数
inputs = []
#从数据库获得路段列表
segmentlist = getsegmentlist()
#任务切割与分配
block_q = separation(len(segmentlist), 90)
while not block_q.empty():
block = block_q.get()
start = block.get('start')
end = block.get('end')
inputs.append({
'seglist': segmentlist[start:end + 1],
'date': date,
'current_time': current_time,
'sourcefolder': sourcefolder,
'resultfolder': resultfolder,
'history': his_df
})
#使用multiprocessing并行进程池来并行处理任务，充分利用cup资源
#确定进程池大小
pool_size = multiprocessing.cpu_count() * 2
print "进程数" + str(pool_size)
# 初始化线程池
pool = multiprocessing.Pool(processes=pool_size,
initializer=start_process,
maxtasksperchild=2)
# 获得各子线程的返回结果，存在列表final_result里
final_result = pool.map(
sonthread,
inputs,
)
#关闭进程池，阻止其他进程进入，等待各个任务完成
pool.close()
#将进程池join到父线程里
pool.join()
pool.terminate()
#父线程在所有子线程结束后，把结果取出来整合
for list in final_result:
mac_d = dict(mac_d, **list[0])
speed_d = dict(speed_d, **list[1])
state_d = dict(state_d, **list[2])
congestion_index_d = dict(congestion_index_d, **list[3])
#插入速度，流量，拥堵状态数据
print speed_d
print mac_d
print state_d
print congestion_index_d
insert_maccount(mac_d)
insert_speed(speed_d)
insert_state(state_d)
insert_congestion_index(congestion_index_d)
#计算交通指数和全网拥堵比率
traffic_index = get_traffic_index(timestr)
congrestion_rate = get_congrestion_rate(timestr)
traffic_state_dict = {
'time': timestr,
'traffic_index': traffic_index,
'congrestion_rate': congrestion_rate
}
#获取本路网涉及到的城市行政区划列表
area_name = getAreas()
for area in area_name:
area_dict = {area: get_traffic_index_of_area(timestr, area)}
traffic_state_dict = dict(traffic_state_dict, **area_dict)
print traffic_state_dict
#插入交通状况记录
insert_traffic_state(traffic_state_dict)
#每轮运算耗时与父线程休眠时间控制
time_cost = time.time() - start_time
print "用时" + str(time_cost)
sleeptime = get_update_time_interval()['update_time'] - time_cost
if sleeptime < 0:
sleeptime = 0
time.sleep(sleeptime)
current_time = current_time + get_update_time_interval()['update_time']
#coding=utf-8
import os
from mysql_utils import getsegmentlist
from getpath import get_current_dir
#从数据库读所有需要计算的路段信息
seglist = getsegmentlist()
#轮询路段列表并创建结果输出目录
for seg in seglist:
print seg
if not os.path.exists(get_current_dir() + '/data/wifiresult/' +
str(seg['segmentid'])):
os.makedirs(get_current_dir() + '/data/wifiresult/' +
str(seg['segmentid']))