value = 0
data = data[6:]
data = sorted(data,
key=lambda x: datetime.strptime(x[1], '%Y/%m/%d').date())
current_date = data[0][1]
output_data = []
village_data = {}
village_values = {}
for item in data:
print(item[1])
key = item[2].replace(' ', '') + item[3]
if key not in village_values:
village_values[key] = 0
event_date = item[1]
if current_date == event_date:
value += 1
village_values[key] += 1
else:
output_data.append(format_data(current_date, value))
insert_village_data(village_data, village_values, current_date)
current_date = event_date
value = 1
village_values = {}
output_data.append(format_data(current_date, value))
insert_village_data(village_data, village_values, current_date)
json_io.write_json('../data/village_bar_data.json', village_data)
json_io.write_json('../data/bar_data.json', output_data)
print(output_data[-1], 'done.')
precp += float(tds[10].text)
precp_count += 1
except:
pass
try:
temp = round(temp / temp_count, 2)
rh = round(rh / rh_count, 2)
precp = round(precp)
except:
return temp, rh, precp
return temp, rh, precp
if __name__ == '__main__':
url_t = "http://e-service.cwb.gov.tw/HistoryDataQuery/DayDataController.do?command=viewMain&station=467410&datepicker="
now = datetime.strptime('2015/10/20', '%Y/%m/%d').date()
end = datetime.now().date()
data = json_io.read_json('../data/weather.json')
while now < end:
html_doc_t = requests.get(url_t + now.strftime('%Y-%m-%d')).text
temp_t, rh_t, precp_t = get_values(html_doc_t)
data[now.strftime('%Y/%m/%d')] = {
'氣溫': temp_t,
'相對溼度': rh_t,
'降水量': precp_t
}
now += timedelta(days=1)
json_io.write_json('../data/weather.json', data)
'Vietnam': './data/Database- VN - Project VN 2.csv',
'South Korea': './data/Database- SK - Project SK 2.csv'
}
stats = {}
output = []
for c, path in files.items():
stats[c] = {}
data = csv_io.read_csv(path)
for row in data[1:]:
if not row[4] or not row[6]:
continue
# "Financier's countries"
if row[4] not in stats[c]:
stats[c][row[4]] = 0
stats[c][row[4]] += float(row[6])
for target, sourceObj in stats.items():
for source_name, v in sourceObj.items():
if source_name != target:
if source_name == 'south Korea':
source_name = 'South Korea'
output.append({
'source': source_name,
'target': target,
'value': round(v, 0)
})
json_io.write_json('./data/flow.json', output)
region = [region[4]]
if code in data:
total = data[code]
elder = elderData[code]
region.append(total)
region.append(elder)
max_tr = round(float(total) / float(people) * 100, 3)
region.append(max_tr)
max_er = round(float(elder) / float(people) * 100, 3)
region.append(max_er)
if max_tr > max_t and int(people) > 20:
max_t = max_tr
if max_er > max_e and int(people) > 20:
max_e = max_er
else:
region.append(0)
region.append(0)
region.append(0)
region.append(0)
new_data[code] = {}
for i in range(0, len(header)):
new_data[code][header[i]] = region[i]
new_data['max_e'] = max_e
new_data['max_t'] = max_t
print(max_e, max_t)
#csv_io.write_csv('../data/small_region_dengue_pop.csv', population)
json_io.write_json('../data/small_region_dengue_pop.json', new_data)
data = filter_data(data)
data = data[1:]
now = '2015年' + data[1][0]
now = datetime.strptime(now, '%Y年%m月%d日').date()
end = '2015年' + data[-1][0]
end = datetime.strptime(end, '%Y年%m月%d日').date() + timedelta(days=1)
header = ['日期', 'Latitude', 'Longitude']
output_data = {}
while now < end:
data_tmp = []
for row in data:
d = '2015年' + row[0]
event_date = datetime.strptime(d, '%Y年%m月%d日').date()
if event_date > now:
break
delta = now - event_date
try:
row[-1], row[-2] = float(row[-1]), float(row[-2])
except ValueError:
continue
if delta.days < 7:
data_tmp.append(row)
data_tmp.insert(0, header)
output_data[now.strftime('%Y/%m/%d')] = data_tmp
now += timedelta(days=1)
json_io.write_json('../data/drug_data.json', output_data)
print(data_tmp, 'done')
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
import requests
import csv
from datetime import datetime
from datetime import timedelta
from datetime import date
from geopy.distance import vincenty
from lib import json_io
from lib import csv_io
if __name__ == '__main__':
data = csv_io.read_csv('./2015_dengue.csv')
data = data[1:]
count = 0
new_data = []
current_date = data[0][0]
for row in data:
event_date = row[0]
if current_date == event_date:
count += 1
else:
new_data.append({'date': current_date, 'value': count})
count = 1
current_date = event_date
new_data.append({'date': current_date, 'value': count})
json_io.write_json('../../data/2015_kao_bar.json', new_data)
if __name__ == '__main__':
url = 'http://denguefever.csie.ncku.edu.tw/file/dengue_all_v3.csv'
data = csv_io.req_csv(url, 'utf-8')
output_data = json_io.read_json('../data/dynamic/data.json')
#output_data = {}
header = data[0]
data = data[1:]
data = sorted(data,
key=lambda x: datetime.strptime(x[1], '%Y/%m/%d').date())
now = datetime.strptime(sys.argv[1], '%Y/%m/%d').date()
end = datetime.strptime(data[-1][1], '%Y/%m/%d').date() + timedelta(days=1)
while now < end:
print(now)
days5, days3, days1 = getDaysWithin(data)
days1 = geo.get_hot_points(days1, 3, 50)
days3 = geo.get_hot_points(days3, 3, 50)
days5 = geo.get_hot_points(days5, 3, 50)
days1.insert(0, header)
days3.insert(0, header)
days5.insert(0, header)
output_data[now.strftime('%Y/%m/%d')] = {}
output_data[now.strftime('%Y/%m/%d')]['one'] = days1
output_data[now.strftime('%Y/%m/%d')]['three'] = days3
output_data[now.strftime('%Y/%m/%d')]['five'] = days5
now += timedelta(days=1)
output_data['end'] = end.strftime('%Y/%m/%d')
json_io.write_json('../data/dynamic/data.json', output_data)
from bs4 import BeautifulSoup
def is_match_country (title, countries):
for country in countries:
if len(country) == 3 and title.startswith(country):
return country
elif country in title:
return country
return False
html_doc = requests.get('http://www.bbc.com/zhongwen/trad/world').text
soup = BeautifulSoup(html_doc)
articles = soup.select(".top-stories__stories .hard-news-unit")
country_list = json_io.read_json('./data/tw.json')
'''lang = json_io.read_json('./data/lang.json')
country_list = []
for en, country in lang.iteritems():
if country['tw']:
country_list.append(country['tw'])
json_io.write_json('tw.json', country_list)'''
geo = json_io.read_json('./data/latlng.json')
pages = []
has_counties = []
for article in articles:
title_link = article.select('.hard-news-unit__headline a')
if title_link:
title = title_link[0].text.replace(' ', '').replace('\n', '')
country = is_match_country(title, country_list)
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from lib import json_io
import geocoder
import time
lang = json_io.read_json('./data/lang.json')
country_dic = {}
for en, country in iter(lang.items()):
name = country['tw']
if name:
latlng = geocoder.google(en).latlng
country_dic[name] = latlng
time.sleep(1)
json_io.write_json('latlng.json', country_dic)
village[key][row[-3]] = {}
village[key][row[-3]]['dengue'] = 1
village[key][row[-3]]['village'] = row[-4]
village[key][row[-3]]['region'] = row[-5]
village[key][row[-3]]['lat'] = row[-2]
village[key][row[-3]]['lng'] = row[-1]
else:
village[key][row[-3]]['dengue'] += 1
if village[key][row[-3]]['village'] != row[-4]:
village[key][row[-3]]['village'] += ', ' + row[-4]
data = {}
for v in village:
max_region = ''
max_number = 0
for region in village[v]:
if village[v][region]['dengue'] > max_number:
max_number = village[v][region]['dengue']
max_region = region
data[max_region] = village[v][max_region]
for region in population:
code = region[2]
people = region[4]
if code in data:
data[code]['ratio'] = float(data[code]['dengue']) / int(people)
data[code]['people'] = people
json_io.write_json('../data/tainan_height_ratio_village.json', data)
data = data[20:]
start_date = data[-1][1] - timedelta(days=13)
days_7ago = data[-1][1] - timedelta(days=6)
end_date = data[-1][1]
day_1_7_values = {}
day_8_14_values = {}
village_values = {}
count = 0
print(start_date, days_7ago, end_date)
for item in data:
if item[1] < start_date or '區' in item[3]:
continue
key = item[2].replace(' ', '') + item[3]
if item[1] < days_7ago:
add_up(key, day_8_14_values)
else:
add_up(key, day_1_7_values)
add_up(key, village_values)
village_rank = sorted(village_values.items(), key=lambda x: x[1])
village_data = create_village_data(village_rank, village_values, \
day_1_7_values, day_8_14_values)
village_data['updateAt'] = end_date.strftime('%Y/%m/%d')
json_io.write_json('../data/village_class.json', village_data)
#! /usr/bin/env python
# -*- coding: utf-8 -*-
from lib import json_io
import os
base_dir = os.path.dirname(os.path.abspath(__file__))
os.chdir(base_dir)
import sys
data1 = json_io.read_json('./data1.json')['result']['records']
data2 = json_io.read_json('./data2.json')['result']['records']
data3 = json_io.read_json('./data3.json')['result']['records']
data4 = json_io.read_json('./data4.json')['result']['records']
data = data1 + data2 + data3 + data4
new_data = {}
for row in data:
if row['SiteName'] not in new_data:
new_data[row['SiteName']] = { \
'city': row['County'], \
'riverName"': row['River'], \
'date': row['SampleDate'], \
'basin': row['Basin'] \
}
prop = row['ItemEngAbbreviation']
value = row['ItemValue']
new_data[row['SiteName']][prop] = value
print (new_data.keys())
json_io.write_json('./data/river.json', new_data)
village_table.append([key])
for item in data:
key = item[1].replace(' ', '') + item[2]
if ('2015/07' in item[0]):
continue
if key not in village_values:
village_values[key] = 0
event_date = item[0]
if current_date == event_date:
value += 1
village_values[key] += 1
else:
output_data.append(format_data(current_date, value))
insert_village_data(village_data, village_values, current_date)
insert_to_village_table(village_values)
current_date = event_date
value = 1
village_values = {}
output_data.append(format_data(current_date, value))
insert_village_data(village_data, village_values, current_date)
insert_to_village_table(village_values)
json_io.write_json('../data/dynamic/village_bar_data_v2.json',
village_data)
json_io.write_json('../data/dynamic/bar_data_v2.json', output_data)
print(output_data[-1], 'done.')
count = 0
for region in data:
sum_before1 = sum_range(data[region], '2015/08/01', '2015/08/13')
sum_before1_avg = sum_before1 / 12
sum_after1 = sum_range(data[region], '2015/08/14', '2015/08/26')
sum_after1_avg = sum_after1 / 12
sum_before2 = sum_range(data[region], '2015/08/21', '2015/08/28')
sum_before2_avg = sum_before2 / 7
sum_after2 = sum_range(data[region], '2015/08/29', '2015/09/06')
sum_after2_avg = sum_after2 / 7
if (sum_after1_avg -
sum_before1_avg) > 0.1 and sum_after2_avg > sum_before2_avg:
count += 1
new_data[region] = {}
new_data[region]['dengue'] = len(data[region])
new_data[region]['region'] = data[region][0][-5]
new_data[region]['village'] = data[region][0][-4]
if (sum_before1 + sum_before2) != 0:
new_data[region]['ratio'] = (sum_after1 + sum_after2) / (
sum_before1 + sum_before2)
else:
new_data[region]['ratio'] = ''
print(count)
json_io.write_json('../data/tainan_rain_rise_region.json', new_data)
village[key][row[-3]]['dengue'] = 1
village[key][row[-3]]['village'] = row[-4]
village[key][row[-3]]['region'] = row[-5]
village[key][row[-3]]['lat'] = row[-2]
village[key][row[-3]]['lng'] = row[-1]
village[key][row[-3]]['date'] = row[1]
else:
village[key][row[-3]]['dengue'] += 1
if village[key][row[-3]]['village'] != row[-4]:
village[key][row[-3]]['village'] += ', ' + row[-4]
data = {}
for v in village:
max_region = ''
max_number = 0
for region in village[v]:
if village[v][region]['dengue'] > max_number:
max_number = village[v][region]['dengue']
max_region = region
data[max_region] = village[v][max_region]
for region in population:
code = region[2]
people = region[4]
if code in data:
data[code]['ratio'] = float(data[code]['dengue']) / int(people)
data[code]['people'] = people
json_io.write_json('../data/tainan_early_region_village.json', data)
