def weather_stations(target_station, prd):
wbans = [wban for (usaf, wban) in TARGET_ISD_STATIONS]
usafs = [usaf for (usaf, wban) in TARGET_ISD_STATIONS]
weather_stations = isd.get_station_list(
'MN', lambda wst: wst.wban in wbans and wst.usaf in usafs)
return isd.find_nearby_station(target_station.lat, target_station.lon,
prd.start_date.date(), weather_stations)
def import_yearly_data(year):
"""
:type year: int
:rtype: list[dict]
"""
logger = getLogger(__name__)
isd_stations = isd.get_station_list(STATE)
logger.debug('loading NOAA ISD data for %d' % year)
res = []
for (usaf, wban) in TARGET_ISD_STATIONS:
st = _isd_station(usaf, wban, isd_stations)
logger.debug(' : ISD station : %s-%s, %s (begin=%s, end=%s)'
% (st.usaf, st.wban, st.station_name, st.begin, st.end))
stime = time.time()
data_list = list(isd.get_year_data(st, year))
logger.debug(' -> data loaded: elapsed time=%s' % timeutil.human_time(seconds=(time.time() - stime)))
stime = time.time()
is_inserted = _insert_noaa_data(year, usaf, wban, data_list)
logger.debug(' -> data inserted: elapsed time=%s, inserted=%s' % (
timeutil.human_time(seconds=(time.time() - stime)), len(data_list)))
if is_inserted:
res.append({'usaf': usaf, 'wban': wban, 'loaded': len(data_list)})
else:
res.append({'usaf': usaf, 'wban': wban, 'loaded': 0})
return res
def import_daily_data(dt):
"""
:type dt: datetime.datetime
:rtype: list[dict]
"""
logger = getLogger(__name__)
isd_stations = isd.get_station_list(STATE)
logger.debug('loading NOAA ISD data for %s' % dt.strftime('%Y-%m-%d'))
res = []
for (usaf, wban) in TARGET_ISD_STATIONS:
loaded = import_daily_data_for_a_station(dt, usaf, wban, isd_stations)
res.append(loaded)
return res
def import_daily_data_for_a_station(dt, usaf, wban, isd_stations=None):
"""
:type dt: datetime.datetime
:type usaf: str
:type wban: str
:type isd_stations: list
:rtype: dict
"""
if not isd_stations:
isd_stations = isd.get_station_list(STATE)
year = dt.year
st = _isd_station(usaf, wban, isd_stations)
isd_data_list = list(isd.get_day_data(st, dt.year, dt.month, dt.day))
if not isd_data_list:
return {'usaf': usaf, 'wban': wban, 'loaded': 0}
else:
is_inserted = _insert_noaa_data(year, usaf, wban, isd_data_list)
if is_inserted:
return {'usaf': usaf, 'wban': wban, 'loaded': len(isd_data_list)}
else:
return {'usaf': usaf, 'wban': wban, 'loaded': 0}
def categorize(ttri, prd, ttdata, **kwargs):
"""
:type ttri: pyticas_tetres.ttypes.TTRouteInfo
:type prd: pyticas.ttypes.Period
:type ttdata: list[pyticas_tetres.ttypes.TravelTimeInfo]
:rtype: int
"""
lock = kwargs.get('lock', nonop_with())
# prepare : coordinates, target year
lat, lon = route.center_coordinates(ttri.route)
year = prd.start_date.year
# nearby weather station list
all_isd_stations = isd.get_station_list('MN', None, False)
isd_stations = isd.find_nearby_station(lat, lon, prd.start_date.date(),
all_isd_stations)
station_idx = 0
if not isd_stations or not isd_stations[station_idx]:
getLogger(__name__).warn(
'! weather.categorize(): no weather information for TTRI(%d)' %
(ttri.id))
return -1
nearby = isd_stations[station_idx][1]
cprd = prd.clone()
cprd.extend_start_hour(1)
cprd.extend_end_hour(1)
# decide nearby weather station which has data during a given period
# by trying to read weather data
wiss = []
hours = (len(cprd.get_timeline()) * prd.interval) / 60 / 60
da_noaa = NoaaWeatherDataAccess(year)
while True:
nearby = isd_stations[station_idx][1]
distance = isd_stations[station_idx][0]
if distance > WEATHER_STATION_DISTANCE_LIMIT:
nearby = None
break
wis = da_noaa.list_by_period(nearby.usaf, nearby.wban, cprd)
if len(wis) < hours * 0.6:
station_idx += 1
continue
if wis:
break
# if wis:
# wiss.append(wis)
# if len(wiss) >= 3:
# break
# continue
if station_idx >= len(isd_stations) - 1:
nearby = None
break
if not nearby:
getLogger(__name__).warn(
'! weather.categorize(): no weather information for TTRI(%d)' %
(ttri.id))
da_noaa.close_session()
return -1
da_ttw = TTWeatherDataAccess(year)
# avoid to save duplicated data
with lock:
is_deleted = da_ttw.delete_range(ttri.id, prd.start_date, prd.end_date)
if not is_deleted or not da_ttw.commit():
da_ttw.rollback()
da_ttw.close_session() # shared session with `da_noaa`
da_noaa.close_session()
return -1
# insert weather data to database
sidx = 0
dict_data = []
for idx, tti in enumerate(ttdata):
sidx, wd = _find_wd(wis, tti.time, sidx)
if not wd:
getLogger(__name__).warn(
'! weather.categorize(): weather data is not found for (tti.time=%s, usaf=%s, wban=%s)'
% (tti.time, wis[-1].usaf, wis[-1].wban))
continue
dict_data.append({'tt_id': tti.id, 'weather_id': wd.id})
if dict_data:
with lock:
inserted_ids = da_ttw.bulk_insert(dict_data, print_exception=True)
if not inserted_ids or not da_ttw.commit():
getLogger(__name__).warn(
'! weather.categorize() fail to insert categorized data')
da_ttw.rollback()
da_ttw.close_session()
da_noaa.close_session()
return -1
da_noaa.close_session()
da_ttw.close_session()
return len(dict_data)