def __init__(self): # 初始化所有參數
self.db = DBService() # database instance
self.ana = dataAnaly() # dataAnaly instance
self.data = Helper() # url data fetch instance
self.sensor_lst = ['pm2.5', 'pm10', 'temperature', 'humidity']
self.all_id_lst = [] # 所有測站 ID list
self.pm25_lst = [] # pm2.5 lsit
self.pm10_lst = [] # pm10 list
self.t_lst = [] # temperatuer lsit
self.h_lst = [] # humidity list
def __init__(self): # 初始化所有變數
self.db = DBService() # Database instance
self.ana = dataAnaly() # DataAnaly instance
self.all_id_lst = [] # 所有測站 ID list
self.pm25_lst = [] # PM2.5 lsit
self.pm10_lst = [] # PM10 list
self.t_lst = [] # Temperatuer lsit
self.h_lst = [] # Humidity list
self.web_error = [] # Error site Id list
self.id_lst = [] # Useful site Id list
self.dict_pm25 = {} # Site Id -> PM2.5 dict
self.dict_pm10 = {} # Site Id -> PM10 dict
class main:
def __init__(self): # 初始化所有參數
self.db = DBService() # database instance
self.ana = dataAnaly() # dataAnaly instance
self.data = Helper() # url data fetch instance
self.sensor_lst = ['pm2.5', 'pm10', 'temperature', 'humidity']
self.all_id_lst = [] # 所有測站 ID list
self.pm25_lst = [] # pm2.5 lsit
self.pm10_lst = [] # pm10 list
self.t_lst = [] # temperatuer lsit
self.h_lst = [] # humidity list
def getInitData(self):
result = self.db.readSiteData() # get all site information
for item in result:
self.all_id_lst.append(item[0])
#get useful data list
self.id_lst, self.url_lst = self.data.urlHelper(
self.all_id_lst, self.sensor_lst)
def getAirValue(self):
self.timeStr = time.strftime('%Y_%m_%d_%H_%M') # get current time
for i in range(0, len(self.url_lst)):
htmlStr = requests.get(self.url_lst[i]).text
tempStr = htmlStr.split(":")[-1]
if i % 4 == 0:
self.pm25_lst.append(int(tempStr[:-3]))
if i % 4 == 1:
self.pm10_lst.append(int(tempStr[:-3]))
if i % 4 == 2:
self.t_lst.append(float(tempStr[:-3]))
if i % 4 == 3:
self.h_lst.append(float(tempStr[:-3]))
def dbHandle(self):
self.db.createAirData(self.timeStr, self.id_lst, self.pm25_lst,
self.pm10_lst, self.t_lst,
self.h_lst) # Create AriInfo table
# self.data = self.db.readAreaData(self.timeStr) # Read AreaInfo table
# 分析錯誤資料 傳回錯誤測站
def analy(self):
self.ana.getAreaId(self.timeStr) # get near area
self.ana.getAreaAirInfo(self.timeStr) # get near area air information
self.ana.s_PM25() # cal near area air PM25 標準差
self.ana.s_PM10() # cal near area air PM10 標準差
self.ana.avg_PM25() # cal near area average PM25
self.ana.avg_PM10() # cal near area average PM10
error_site = self.ana.grubbsTest() # get final error site value
return error_site
def __init__(self):
self.db = DBService() # new database instance
self.ana = dataAnaly() # new dataAnaly instance
class TableService:
# 2017-10-12 add by Mayday
def __init__(self):
self.db = DBService() # new database instance
self.ana = dataAnaly() # new dataAnaly instance
# 2017-10-12 add by Mayday
# <summary> 取得所有空氣資料表格名稱 </summary>
# <return> 所有空氣資料表名稱串列 </return>
def getAllAirInfoTableName(self):
tables = self.db.readAllAirInfoTableName()
result = []
for item in tables:
result.append(item[1])
return result
# 2017-11-17 add by Maydya
# <summary> 取得所有異常資料表名稱 </summary>
# <return> 所有異常資料表名稱串列 </return>
def getAllErrorTableName(self):
tables = self.db.readAllErrorTableName()
result = []
for table in tables:
result.append(table[1])
return result
# 2017-10-12 add by Mayday
# <summary> 取得 x 與 y 軸的數據 </summary>
# <param name = "table_name"> 表格名稱或表格陣列 </param>
# <param name = "stId"> 測站 ID </param>
# <param name = "tag"> 時間模式 </param>
# <return> X 軸數據, y 軸數據 </return>
def getXYAxis(self, table_name, stId, interval):
pm25_lst = [] #Y 軸數據
time_lst = [] #X 軸數據
if interval == 60: # get the timeline of hour
h = int(table_name[-1].split('_')[-2]) # get current hour
for i in range(h - 11, h + 1): # 過去 12 小時到現在做為 X 軸
if (i < 0): # < 0 時往前會出現負數
i = 24 + i
time_lst.append(i)
for i in range(
len(table_name) - 144, # 取得過去 12 小時的空氣資料
len(table_name),
12): # 一小時取一次，間隔 12(5分)
y = self.db.readPM25ById(table_name[i], stId)
if y == None: # 感測器如果關了
pm25_lst.append(-1) # 填入 -1
else:
pm25_lst.append(y[1]) # 填充 Y 軸
elif interval == 30: # get the timeline of 30 min
for i in range(
len(table_name) - 72, # 取得過去 6 小時的空氣資料
len(table_name),
6): # 半小時取一次，間隔 6(5分)
x = table_name[i][-5:].replace( # 取得 X 軸，即時、分
'_', ':')
time_lst.append(x) # 數據填充 X 軸
y = self.db.readPM25ById(
table_name[i], # 取得 Y 軸，即 PM2.5 的值
stId)
if y == None: # 感測器如果關了
pm25_lst.append(-1) # 填入 -1
else:
pm25_lst.append(y[1]) # 數據填充 Y 軸
elif interval == 5: # get the timeline of 5 min
for i in range(
len(table_name) - 12, # 取得過去 1 小時的空氣資料
len(table_name)): # 5 分鐘取一次，間隔 12(1H)
x = table_name[i][-5:].replace( # 取得 X 軸，即時、分
'_', ':')
time_lst.append(x) # 填充 X 軸
y = self.db.readPM25ById(
table_name[i], # 取得 Y 軸，即PM2.5 的值
stId)
if y == None: # 感測器如果關了
pm25_lst.append(-1) # 填入 -1
else:
pm25_lst.append(y[1]) # 填充 Y 軸
else:
x = self.ana.getAreaData(table_name[8:])
temp = []
for item in x:
if stId == item[0]:
temp = item
for item in temp:
y = self.db.readPM25ById(table_name, item)
pm25_lst.append(y[1])
time_lst.append(self.db.readSiteNoteById(item)[0])
return time_lst, pm25_lst
class main:
def __init__(self): # 初始化所有變數
self.db = DBService() # Database instance
self.ana = dataAnaly() # DataAnaly instance
self.all_id_lst = [] # 所有測站 ID list
self.pm25_lst = [] # PM2.5 lsit
self.pm10_lst = [] # PM10 list
self.t_lst = [] # Temperatuer lsit
self.h_lst = [] # Humidity list
self.web_error = [] # Error site Id list
self.id_lst = [] # Useful site Id list
self.dict_pm25 = {} # Site Id -> PM2.5 dict
self.dict_pm10 = {} # Site Id -> PM10 dict
# 2017-11-23 edit by Mayday
# <summary> 初始有效 ID 與爬蟲網址 </summary>
def getInitData(self):
print("strftime")
self.timeStr = time.strftime('%Y_%m_%d_%H_%M') # GET current time
print("db.lst_read")
site_info_lst = self.db.lst_readSiteData() # Get all site information
print("getInitdata_for")
for item in site_info_lst: # Input all_id_lst
self.all_id_lst.append(item[0])
print("get useful data list")
return self.timeStr
# 2017-11-23 edit by Mayday
# <summary> 爬蟲方法 取得測站數據 </summary>
def getAirValue(self):
for i in range(0, len(self.all_id_lst)):
try:
res1 = urlopen(
Request(
"http://www.airq.org.tw/Home/GetCurrentValueApi?station={}&sensor=pm2.5"
.format(self.all_id_lst[i])))
res2 = urlopen(
Request(
"http://www.airq.org.tw/Home/GetCurrentValueApi?station={}&sensor=pm10"
.format(self.all_id_lst[i])))
res3 = urlopen(
Request(
"http://www.airq.org.tw/Home/GetCurrentValueApi?station={}&sensor=temperature"
.format(self.all_id_lst[i])))
res4 = urlopen(
Request(
"http://www.airq.org.tw/Home/GetCurrentValueApi?station={}&sensor=humidity"
.format(self.all_id_lst[i])))
except IOError:
self.web_error.append(self.all_id_lst[i])
else:
pm25_pack = json.loads(res1.read())[0]['text'][14:]
pm10_pack = json.loads(res2.read())[0]['text'][14:]
temperature_pack = json.loads(res3.read())[0]['text'][14:]
humidity_pack = json.loads(res4.read())[0]['text'][14:]
self.id_lst.append(self.all_id_lst[i])
self.pm25_lst.append(int(pm25_pack))
self.pm10_lst.append(int(pm10_pack))
self.h_lst.append(float(humidity_pack))
self.t_lst.append(float(temperature_pack))
# id -> pm25
self.dict_pm25 = dict(zip(self.id_lst, self.pm25_lst))
# id -> pm10
self.dict_pm10 = dict(zip(self.id_lst, self.pm10_lst))
# 2017-11-23 edit by Mayday
# <summary> 儲存測站數據 </summary>
# <param name="timeStr"> 表格名稱 </summary>
# <param name="id_lst"> 所有測站 ID 串列 </param>
# <param name="pm25_lst"> 所有測站 PM2.5 數據串列 </param>
# <param name="pm10_lst"> 所有測站 PM10 數據串列 </param>
# <param name="t_lst"> 所有測站溫度數據串列 </param>
# <param name="h_lst"> 所有測站濕度數據串列 </param>
def dataSave(self):
self.db.createAirData(self.timeStr, self.id_lst, self.pm25_lst,
self.pm10_lst, self.t_lst,
self.h_lst) # Create AriInfo table
# self.data = self.db.readAreaData(self.timeStr) # Read AreaInfo table
# 分析錯誤資料 傳回錯誤測站
def analy(self):
print("ana.getAreaId")
self.ana.getAreaId(self.timeStr) # Get near area
print("ana.getAreaAirInfo")
self.ana.getAreaAirInfo(
self.timeStr, self.dict_pm25,
self.dict_pm10) # Get near area PM25 and PM10 information
print("s_PM25")
self.ana.s_PM25() # Cal near area air PM25 標準差
print("s_PM10")
self.ana.s_PM10() # Cal near area air PM10 標準差
print("avg_PM25")
self.ana.avg_PM25() # Cal near area average PM25
print("avg_PM10")
self.ana.avg_PM10() # Cal near area average PM10
print("GrubbsTest")
error_site = self.ana.grubbsTest() # Get final error site value
return error_site
from django.shortcuts import render
from django.http import HttpResponse
from django.http import JsonResponse
from datetime import datetime # 時間
from GPMSWEB.DBService import DBService # 自建資料庫類別
from GPMSWEB.TableService import TableService # 自建資料表類別
from GPMSWEB.dataAnaly import dataAnaly # 資料分析類別
from firebase import firebase
# Create your views here.
# 初始所有類別物件 全域
db = DBService() # 資料庫物件
analy = dataAnaly() # 資料分析物件
table = TableService() # 表格資料物件
def index(requests):
error_table_name_lst = table.lst_getAllErrorTableName()
print(error_table_name_lst[-1])
error = db.readErrorData(error_table_name_lst[-1])
date = error_table_name_lst[-1][10:20].replace("_","-")
time = error_table_name_lst[-1][-5:].replace("_",":")
return render(requests, "index.html", locals())
# 取得所有測站資料 傳送至 data.html 頁面
def data(requests):
if(len(requests.GET) == 0):
row = 0
else:
row = int(requests.GET['selectedIndex'])
def __init__(self):
self.db = DBService() # Database instance
self.area_gps = [] # Final site area list
self.note = [] # Site name list
class dataAnaly:
# 2017-10-20 add by Mayday
def __init__(self):
self.db = DBService() # Database instance
self.area_gps = [] # Final site area list
self.note = [] # Site name list
# 2017-11-23 edit by Mayday
# <summary>Get area point id</summary>
# <param name = "timeStr">Table name</param>
def getAreaId(self, timeStr):
gps = [] # Site area temp list
print("read_DB_Postition")
site_position_lst = self.db.lst_readPositionData(
timeStr) # Read site position data
print("add_Position_to_GPS[]")
for item in site_position_lst:
gps.append([item[0], item[1], item[2]]) # Add site id,lat and lon
self.note.append(item[3]) # Add site name
# cal area point
for i in gps:
temp = [] # Temp list
temp.append(i[0]) # Add point itself
print("add_area_function")
for j in gps:
if (self.haversine(i[2], i[1], j[2], j[1]) > 0
and self.haversine(i[2], i[1], j[2], j[1]) <= 5):
temp.append(j[0])
self.area_gps.append(temp)
# 2017-11-24 edit by Mayday
# <summary> 取得區域測站名稱 </summary>
# <return> 區域測站名稱串列 </return>
def lst_getAreaSite(self, row):
st_note_lst = []
for item in self.area_gps[row]:
st_note = self.db.m_readSiteNoteById(item)
st_note_lst.append(st_note[0])
return st_note_lst
# <summary>計算附近的點</summary>
# <param name = "lon1">第一點經度</param>
# <param name = "lat1">第一點緯度</param>
# <param name = "lon2">第二點經度</param>
# <param name = "lat2">第二點緯度</param>
def haversine(self, lon1, lat1, lon2, lat2): # 經度1，緯度1，經度2，緯度2
lon1, lat1, lon2, lat2 = map(math.radians, [lon1, lat1, lon2, lat2])
# haversine
dlon = lon2 - lon1
dlat = lat2 - lat1
a = math.sin(dlat / 2)**2 + math.cos(lat1) * math.cos(lat2) * math.sin(
dlon / 2)**2
c = 2 * math.asin(math.sqrt(a))
r = 6371 # 地球半徑
return c * r
# 2017-11-23 edit by Mayday
# <summary>Get area PM2.5 and PM10 information</summary>
# <param name = "timeStr">Table name</param>
def getAreaAirInfo(self, timeStr, dict_pm25, dict_pm10):
self.area_pm25 = [] # Area PM2.5 list
self.area_pm10 = [] # Area PM10 list
for item in self.area_gps:
print("getAreaAirInfo-1")
temp25 = [] # Temp PM2.5 list
temp10 = [] # Temp PM10 list
for Id in item:
print("getAreaAirInfo-2")
temp25.append(dict_pm25['{}'.format(Id)])
temp10.append(dict_pm10['{}'.format(Id)])
self.area_pm25.append(temp25)
self.area_pm10.append(temp10)
#return self.neighbor_pm25,self.neighbor_pm10
# PM25 標準差
def s_PM25(self):
self.s_pm25_lst = []
for item in self.area_pm25:
self.s_pm25_lst.append(np.std(item))
# PM10 標準差
def s_PM10(self):
self.s_pm10_lst = []
for item in self.area_pm10:
self.s_pm10_lst.append(np.std(item))
# PM25 Average
def avg_PM25(self):
self.avg_pm25_lst = []
for item in self.area_pm25:
self.avg_pm25_lst.append(np.mean(item))
# PM10 Average
def avg_PM10(self):
self.avg_pm10_lst = []
for item in self.area_pm10:
self.avg_pm10_lst.append(np.mean(item))
# Cal grubbsTest value
def grubbsTest(self):
print("grubbsTest-1")
self.area_final = []
index = 0 # item index
print("grubbsTest-2")
for item in self.area_pm25:
An = self.db.m_selectGAlpha(len(item))
if An != None: # if N >= 3
x = An[0]
Gn = abs((item[0] - self.avg_pm25_lst[index]) /
self.s_pm25_lst[index])
final = x - Gn
if final < 0:
self.area_final.append(self.note[index])
index += 1
print("grubbsTest-3")
return self.area_final
class dataAnaly:
# 2017-10-20 add by Mayday
def __init__(self):
self.db = DBService() # Database instance
self.area_gps = [] # Final site area list
self.note = [] # Site name list
# <summary>Get area point id</summary>
# <param name = "timeStr">Table name</param>
def getAreaId(self, timeStr):
gps = [] # Site area temp list
r_data = self.db.readAreaData(timeStr) # Read area data from db
for item in r_data:
gps.append([item[0], item[1], item[2]]) # Add site id,lat and lon
self.note.append(item[3]) # Add site name
#cal area point
for i in gps:
temp = [] # Temp list
temp.append(i[0]) # Add point itself
for j in gps:
if (self.haversine(i[2], i[1], j[2], j[1]) > 0
and self.haversine(i[2], i[1], j[2], j[1]) <= 7):
temp.append(j[0])
self.area_gps.append(temp)
# 2017-10-25 add by Mayday
# <summary> 取得區域測站名稱 </summary>
# <return> 區域測站名稱串列 </return>
def getAreaSite(self, row):
result = []
for item in self.area_gps[row]:
print(item)
st_note = self.db.readSiteNoteById(item)
result.append(st_note[0])
print(result)
return result
# <summary>計算附近的點</summary>
# <param name = "lon1">第一點經度</param>
# <param name = "lat1">第一點緯度</param>
# <param name = "lon2">第二點經度</param>
# <param name = "lat2">第二點緯度</param>
def haversine(self, lon1, lat1, lon2, lat2): # 經度1，緯度1，經度2，緯度2
lon1, lat1, lon2, lat2 = map(math.radians, [lon1, lat1, lon2, lat2])
# haversine
dlon = lon2 - lon1
dlat = lat2 - lat1
a = math.sin(dlat / 2)**2 + math.cos(lat1) * math.cos(lat2) * math.sin(
dlon / 2)**2
c = 2 * math.asin(math.sqrt(a))
r = 6371 # 地球半徑
return c * r
# <summary>Get area PM2.5 and PM10 information</summary>
# <param name = "timeStr">Table name</param>
def getAreaAirInfo(self, timeStr):
self.area_pm25 = [] # Area PM2.5 list
self.area_pm10 = [] # Area PM10 list
for item in self.area_gps:
temp25 = [] # Temp PM2.5 list
temp10 = [] # Temp PM10 list
for i in item:
r_data = self.db.readPM25PM10(timeStr, i)
temp25.append(r_data[0][0])
temp10.append(r_data[0][1])
self.area_pm25.append(temp25)
self.area_pm10.append(temp10)
#return self.neighbor_pm25,self.neighbor_pm10
# PM25 標準差
def s_PM25(self):
self.s_pm25_lst = []
for item in self.area_pm25:
self.s_pm25_lst.append(np.std(item))
# PM10 標準差
def s_PM10(self):
self.s_pm10_lst = []
for item in self.area_pm10:
self.s_pm10_lst.append(np.std(item))
# PM25 Average
def avg_PM25(self):
self.avg_pm25_lst = []
for item in self.area_pm25:
self.avg_pm25_lst.append(np.mean(item))
# PM10 Average
def avg_PM10(self):
self.avg_pm10_lst = []
for item in self.area_pm10:
self.avg_pm10_lst.append(np.mean(item))
# cal grubbsTest value
def grubbsTest(self):
self.area_final = []
index = 0 # item index
for item in self.area_pm25:
An = self.db.selectGAlpha(len(item))
if An != None: # if N >= 3
x = An[0]
Gn = abs((item[0] - self.avg_pm25_lst[index]) /
self.s_pm25_lst[index])
final = x - Gn
if final < 0:
self.area_final.append(self.note[index])
index += 1
return self.area_final