value = 0 data = data[6:] data = sorted(data, key=lambda x: datetime.strptime(x[1], '%Y/%m/%d').date()) current_date = data[0][1] output_data = [] village_data = {} village_values = {} for item in data: print(item[1]) key = item[2].replace(' ', '') + item[3] if key not in village_values: village_values[key] = 0 event_date = item[1] if current_date == event_date: value += 1 village_values[key] += 1 else: output_data.append(format_data(current_date, value)) insert_village_data(village_data, village_values, current_date) current_date = event_date value = 1 village_values = {} output_data.append(format_data(current_date, value)) insert_village_data(village_data, village_values, current_date) json_io.write_json('../data/village_bar_data.json', village_data) json_io.write_json('../data/bar_data.json', output_data) print(output_data[-1], 'done.')
precp += float(tds[10].text) precp_count += 1 except: pass try: temp = round(temp / temp_count, 2) rh = round(rh / rh_count, 2) precp = round(precp) except: return temp, rh, precp return temp, rh, precp if __name__ == '__main__': url_t = "http://e-service.cwb.gov.tw/HistoryDataQuery/DayDataController.do?command=viewMain&station=467410&datepicker=" now = datetime.strptime('2015/10/20', '%Y/%m/%d').date() end = datetime.now().date() data = json_io.read_json('../data/weather.json') while now < end: html_doc_t = requests.get(url_t + now.strftime('%Y-%m-%d')).text temp_t, rh_t, precp_t = get_values(html_doc_t) data[now.strftime('%Y/%m/%d')] = { '氣溫': temp_t, '相對溼度': rh_t, '降水量': precp_t } now += timedelta(days=1) json_io.write_json('../data/weather.json', data)
'Vietnam': './data/Database- VN - Project VN 2.csv', 'South Korea': './data/Database- SK - Project SK 2.csv' } stats = {} output = [] for c, path in files.items(): stats[c] = {} data = csv_io.read_csv(path) for row in data[1:]: if not row[4] or not row[6]: continue # "Financier's countries" if row[4] not in stats[c]: stats[c][row[4]] = 0 stats[c][row[4]] += float(row[6]) for target, sourceObj in stats.items(): for source_name, v in sourceObj.items(): if source_name != target: if source_name == 'south Korea': source_name = 'South Korea' output.append({ 'source': source_name, 'target': target, 'value': round(v, 0) }) json_io.write_json('./data/flow.json', output)
region = [region[4]] if code in data: total = data[code] elder = elderData[code] region.append(total) region.append(elder) max_tr = round(float(total) / float(people) * 100, 3) region.append(max_tr) max_er = round(float(elder) / float(people) * 100, 3) region.append(max_er) if max_tr > max_t and int(people) > 20: max_t = max_tr if max_er > max_e and int(people) > 20: max_e = max_er else: region.append(0) region.append(0) region.append(0) region.append(0) new_data[code] = {} for i in range(0, len(header)): new_data[code][header[i]] = region[i] new_data['max_e'] = max_e new_data['max_t'] = max_t print(max_e, max_t) #csv_io.write_csv('../data/small_region_dengue_pop.csv', population) json_io.write_json('../data/small_region_dengue_pop.json', new_data)
data = filter_data(data) data = data[1:] now = '2015年' + data[1][0] now = datetime.strptime(now, '%Y年%m月%d日').date() end = '2015年' + data[-1][0] end = datetime.strptime(end, '%Y年%m月%d日').date() + timedelta(days=1) header = ['日期', 'Latitude', 'Longitude'] output_data = {} while now < end: data_tmp = [] for row in data: d = '2015年' + row[0] event_date = datetime.strptime(d, '%Y年%m月%d日').date() if event_date > now: break delta = now - event_date try: row[-1], row[-2] = float(row[-1]), float(row[-2]) except ValueError: continue if delta.days < 7: data_tmp.append(row) data_tmp.insert(0, header) output_data[now.strftime('%Y/%m/%d')] = data_tmp now += timedelta(days=1) json_io.write_json('../data/drug_data.json', output_data) print(data_tmp, 'done')
#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- import requests import csv from datetime import datetime from datetime import timedelta from datetime import date from geopy.distance import vincenty from lib import json_io from lib import csv_io if __name__ == '__main__': data = csv_io.read_csv('./2015_dengue.csv') data = data[1:] count = 0 new_data = [] current_date = data[0][0] for row in data: event_date = row[0] if current_date == event_date: count += 1 else: new_data.append({'date': current_date, 'value': count}) count = 1 current_date = event_date new_data.append({'date': current_date, 'value': count}) json_io.write_json('../../data/2015_kao_bar.json', new_data)
if __name__ == '__main__': url = 'http://denguefever.csie.ncku.edu.tw/file/dengue_all_v3.csv' data = csv_io.req_csv(url, 'utf-8') output_data = json_io.read_json('../data/dynamic/data.json') #output_data = {} header = data[0] data = data[1:] data = sorted(data, key=lambda x: datetime.strptime(x[1], '%Y/%m/%d').date()) now = datetime.strptime(sys.argv[1], '%Y/%m/%d').date() end = datetime.strptime(data[-1][1], '%Y/%m/%d').date() + timedelta(days=1) while now < end: print(now) days5, days3, days1 = getDaysWithin(data) days1 = geo.get_hot_points(days1, 3, 50) days3 = geo.get_hot_points(days3, 3, 50) days5 = geo.get_hot_points(days5, 3, 50) days1.insert(0, header) days3.insert(0, header) days5.insert(0, header) output_data[now.strftime('%Y/%m/%d')] = {} output_data[now.strftime('%Y/%m/%d')]['one'] = days1 output_data[now.strftime('%Y/%m/%d')]['three'] = days3 output_data[now.strftime('%Y/%m/%d')]['five'] = days5 now += timedelta(days=1) output_data['end'] = end.strftime('%Y/%m/%d') json_io.write_json('../data/dynamic/data.json', output_data)
from bs4 import BeautifulSoup def is_match_country (title, countries): for country in countries: if len(country) == 3 and title.startswith(country): return country elif country in title: return country return False html_doc = requests.get('http://www.bbc.com/zhongwen/trad/world').text soup = BeautifulSoup(html_doc) articles = soup.select(".top-stories__stories .hard-news-unit") country_list = json_io.read_json('./data/tw.json') '''lang = json_io.read_json('./data/lang.json') country_list = [] for en, country in lang.iteritems(): if country['tw']: country_list.append(country['tw']) json_io.write_json('tw.json', country_list)''' geo = json_io.read_json('./data/latlng.json') pages = [] has_counties = [] for article in articles: title_link = article.select('.hard-news-unit__headline a') if title_link: title = title_link[0].text.replace(' ', '').replace('\n', '') country = is_match_country(title, country_list)
#!/usr/bin/env python2.7 # -*- coding: utf-8 -*- from lib import json_io import geocoder import time lang = json_io.read_json('./data/lang.json') country_dic = {} for en, country in iter(lang.items()): name = country['tw'] if name: latlng = geocoder.google(en).latlng country_dic[name] = latlng time.sleep(1) json_io.write_json('latlng.json', country_dic)
village[key][row[-3]] = {} village[key][row[-3]]['dengue'] = 1 village[key][row[-3]]['village'] = row[-4] village[key][row[-3]]['region'] = row[-5] village[key][row[-3]]['lat'] = row[-2] village[key][row[-3]]['lng'] = row[-1] else: village[key][row[-3]]['dengue'] += 1 if village[key][row[-3]]['village'] != row[-4]: village[key][row[-3]]['village'] += ', ' + row[-4] data = {} for v in village: max_region = '' max_number = 0 for region in village[v]: if village[v][region]['dengue'] > max_number: max_number = village[v][region]['dengue'] max_region = region data[max_region] = village[v][max_region] for region in population: code = region[2] people = region[4] if code in data: data[code]['ratio'] = float(data[code]['dengue']) / int(people) data[code]['people'] = people json_io.write_json('../data/tainan_height_ratio_village.json', data)
data = data[20:] start_date = data[-1][1] - timedelta(days=13) days_7ago = data[-1][1] - timedelta(days=6) end_date = data[-1][1] day_1_7_values = {} day_8_14_values = {} village_values = {} count = 0 print(start_date, days_7ago, end_date) for item in data: if item[1] < start_date or '區' in item[3]: continue key = item[2].replace(' ', '') + item[3] if item[1] < days_7ago: add_up(key, day_8_14_values) else: add_up(key, day_1_7_values) add_up(key, village_values) village_rank = sorted(village_values.items(), key=lambda x: x[1]) village_data = create_village_data(village_rank, village_values, \ day_1_7_values, day_8_14_values) village_data['updateAt'] = end_date.strftime('%Y/%m/%d') json_io.write_json('../data/village_class.json', village_data)
#! /usr/bin/env python # -*- coding: utf-8 -*- from lib import json_io import os base_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(base_dir) import sys data1 = json_io.read_json('./data1.json')['result']['records'] data2 = json_io.read_json('./data2.json')['result']['records'] data3 = json_io.read_json('./data3.json')['result']['records'] data4 = json_io.read_json('./data4.json')['result']['records'] data = data1 + data2 + data3 + data4 new_data = {} for row in data: if row['SiteName'] not in new_data: new_data[row['SiteName']] = { \ 'city': row['County'], \ 'riverName"': row['River'], \ 'date': row['SampleDate'], \ 'basin': row['Basin'] \ } prop = row['ItemEngAbbreviation'] value = row['ItemValue'] new_data[row['SiteName']][prop] = value print (new_data.keys()) json_io.write_json('./data/river.json', new_data)
village_table.append([key]) for item in data: key = item[1].replace(' ', '') + item[2] if ('2015/07' in item[0]): continue if key not in village_values: village_values[key] = 0 event_date = item[0] if current_date == event_date: value += 1 village_values[key] += 1 else: output_data.append(format_data(current_date, value)) insert_village_data(village_data, village_values, current_date) insert_to_village_table(village_values) current_date = event_date value = 1 village_values = {} output_data.append(format_data(current_date, value)) insert_village_data(village_data, village_values, current_date) insert_to_village_table(village_values) json_io.write_json('../data/dynamic/village_bar_data_v2.json', village_data) json_io.write_json('../data/dynamic/bar_data_v2.json', output_data) print(output_data[-1], 'done.')
count = 0 for region in data: sum_before1 = sum_range(data[region], '2015/08/01', '2015/08/13') sum_before1_avg = sum_before1 / 12 sum_after1 = sum_range(data[region], '2015/08/14', '2015/08/26') sum_after1_avg = sum_after1 / 12 sum_before2 = sum_range(data[region], '2015/08/21', '2015/08/28') sum_before2_avg = sum_before2 / 7 sum_after2 = sum_range(data[region], '2015/08/29', '2015/09/06') sum_after2_avg = sum_after2 / 7 if (sum_after1_avg - sum_before1_avg) > 0.1 and sum_after2_avg > sum_before2_avg: count += 1 new_data[region] = {} new_data[region]['dengue'] = len(data[region]) new_data[region]['region'] = data[region][0][-5] new_data[region]['village'] = data[region][0][-4] if (sum_before1 + sum_before2) != 0: new_data[region]['ratio'] = (sum_after1 + sum_after2) / ( sum_before1 + sum_before2) else: new_data[region]['ratio'] = '' print(count) json_io.write_json('../data/tainan_rain_rise_region.json', new_data)
village[key][row[-3]]['dengue'] = 1 village[key][row[-3]]['village'] = row[-4] village[key][row[-3]]['region'] = row[-5] village[key][row[-3]]['lat'] = row[-2] village[key][row[-3]]['lng'] = row[-1] village[key][row[-3]]['date'] = row[1] else: village[key][row[-3]]['dengue'] += 1 if village[key][row[-3]]['village'] != row[-4]: village[key][row[-3]]['village'] += ', ' + row[-4] data = {} for v in village: max_region = '' max_number = 0 for region in village[v]: if village[v][region]['dengue'] > max_number: max_number = village[v][region]['dengue'] max_region = region data[max_region] = village[v][max_region] for region in population: code = region[2] people = region[4] if code in data: data[code]['ratio'] = float(data[code]['dengue']) / int(people) data[code]['people'] = people json_io.write_json('../data/tainan_early_region_village.json', data)