def collect(target_station, periods, **kwargs):
""" returns nighttime traffic data
:type target_station: pyticas.ttypes.RNodeObject
:type periods: list[pyticas.ttypes.Period]
:rtype: (list[float], list[float], list[list[float]], list[list[float]], list[pyticas.ttypes.Period])
"""
rdr = ncrtes.get_infra().rdr
dc, valid_detectors = kwargs.get('dc',
None), kwargs.get('valid_detectors', None)
if not dc:
dc, valid_detectors = lane.get_detector_checker(target_station)
if not dc:
return None, None, None, None, None
uss, kss, used_periods = [], [], []
for prd in periods:
us = rdr.get_speed(target_station, prd, dc)
if not ncrtes.get_infra().is_missing(us.data):
ks = rdr.get_density(target_station, prd, dc)
uss.append(us.data)
kss.append(ks.data)
used_periods.append(prd)
if not uss:
return None, None, None, None, None
avg_us = data_util.avg_multi(uss, only_positive=True)
avg_ks = data_util.avg_multi(kss, only_positive=True)
import xlsxwriter
import os
output_path = ncrtes.get_infra().get_path('ncrtes', create=True)
filepath = os.path.join(output_path,
'nighttime-%s.xlsx' % target_station.station_id)
wb = xlsxwriter.Workbook(filepath)
ws = wb.add_worksheet('nsr_data')
ws.write_row(0, 0,
['time', 'avg'] + [prd.get_date_string() for prd in periods])
ws.write_column(1, 0, [
dt.strftime('%H:%M')
for dt in periods[0].get_timeline(as_datetime=True)
])
ws.write_column(1, 1, avg_us)
col = 2
for idx, us in enumerate(uss):
ws.write_column(1, col, us)
col += 1
wb.close()
return avg_us, avg_ks, uss, kss, used_periods
def get_target_stations(corr_name):
""" Returns potential target stations satisfying the following conditions
**Conditions**
- normal station
- not temporary station
- not wavetronics station
- not radar station
- not velocity station
- has target lanes
- lane 2 or lane 3 (when lane 1 is an auxiliary lane)
- not on curve
- the station on curve shows lower-level of U-K comparing to nearby stations
:type corr_name: str
:rtype: list[pyticas.ttypes.RNodeObject]
"""
infra = ncrtes.get_infra()
corr = infra.get_corridor_by_name(corr_name)
stations = []
for st in corr.stations:
if not st.is_normal_station():
continue
if not any(lane.get_target_detectors(st)):
continue
up_station, dn_station = _find_up_and_down_stations(st)
if up_station is not None:
angle = infra.geo.angle_of_rnodes(up_station, st, dn_station)
if angle < WITHRAW_STATION_ANGLE:
continue
stations.append(st)
return stations
def ncrtes_ts_list():
try:
year = int(request.form.get('year'))
corridor_name = request.form.get('corridor_name')
months = get_normal_months_from_year(year)
wsDA = WinterSeasonDataAccess()
wsi = wsDA.get_by_year(year)
if not wsi:
wsi = itypes.WinterSeasonInfo()
wsi.set_months(months)
wsi.name = 'WinterSeason %s-%s' % (months[0][0], months[-1][0])
wsDA.insert(wsi, autocommit=True)
wsDA.close()
tsDA = TargetStationDataAccess()
ts_list = tsDA.list_by_corridor_name(year, corridor_name)
tsDA.close()
snrDA = SnowRouteDataAccess()
snow_routes = snrDA.list_by_year(year)
snrDA.close()
tlcDA = TargetLaneConfigDataAccess()
tlc_list = tlcDA.list_by_corridor_name(year, corridor_name)
tlcDA.close()
# sort (from upstream to downstream)
infra = ncrtes.get_infra()
corr = infra.get_corridor_by_name(corridor_name)
res = []
for idx, st in enumerate(corr.stations):
tsi = _find_target_station_info(st.station_id, ts_list)
tlci = _find_target_lane_info(st.station_id, tlc_list)
if not tsi:
snow_route = _find_snow_route(st.station_id, snow_routes)
print(idx, st.station_id)
tsi = itypes.TargetStationInfo()
tsi.winterseason_id = wsi.id if wsi else None
tsi.station_id = st.station_id
tsi.snowroute_id = snow_route.id if snow_route else None
tsi.snowroute_name = snow_route._snowroute_group.name if snow_route else None
tsi.corridor_name = st.corridor.name
tsi.detectors = tlci.detectors if tlci else ','.join(
[det.name for det in lane.get_target_detectors(st)])
tsi.normal_function_id = None
res.append(tsi)
else:
print(idx, st.station_id)
tsi.detectors = tlci.detectors if tlci else ','.join(
[det.name for det in lane.get_target_detectors(st)])
res.append(tsi)
res = [v for v in res if v.detectors]
except Exception as ex:
return prot.response_error(
'exception occured when retrieving data')
return prot.response_success({'list': res})
def cache_file_path(station_id, months):
""" returns nsr_data path
:type station_id: str
:type months: list[(int, int)]
:rtype: str
"""
# ymds = ['%d%02d' % (y, m) for (y, m) in months]
years = list(set(['%d' % y for (y, m) in months]))
ystr = '-'.join(sorted(years))
path = 'ncrtes/%s/%s' % (ystr, DATA_DIR)
return os.path.join(ncrtes.get_infra().get_path(path, create=True),
'%s.json' % station_id)
def _output_path(prd, corr_name):
infra = ncrtes.get_infra()
output_path = os.path.join(
infra.get_path('ncrtes', create=True), 'wetnormal',
'%s - %s' % (prd.start_date.strftime('%Y-%m-%d'), corr_name))
# output_path = os.path.join(infra.get_path(setting.OUTPUT_DIR, create=True), 'output',
# datetime.datetime.now().strftime('%Y%m%d %H%M%S'))
if not os.path.exists(output_path):
os.makedirs(output_path)
return output_path
def _output_path(*filename):
import os
from pyticas_ncrtes import ncrtes
infra = ncrtes.get_infra()
ncrtesdir = infra.get_path('ncrtes', create=True)
outputdir = os.path.join(ncrtesdir, 'debug')
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if filename:
outputfile = os.path.join(outputdir, *filename)
_outputdir = os.path.dirname(outputfile)
if not os.path.exists(_outputdir):
os.makedirs(_outputdir)
return outputfile
else:
return outputdir
def _station_data(target_station, prd, dc):
""" return q,k,u data for given period
:type target_station: pyticas.ttypes.RNodeObject
:type prd: pyticas.ttypes.Period
:type dc: function
:rtype: (pyticas.ttypes.RNodeData, pyticas.ttypes.RNodeData, pyticas.ttypes.RNodeData)
"""
rdr = ncrtes.get_infra().rdr
logger = getLogger(__name__)
us = rdr.get_speed(target_station, prd, dc)
if not us or _is_missing_day(us.data, prd.interval):
logger.debug('Station %s is missing at %s!!' % (target_station.station_id, prd.get_date_string()))
return None, None, None
ks = rdr.get_density(target_station, prd, dc)
qs = rdr.get_average_flow(target_station, prd, dc)
return us, ks, qs
def _read_snow_uk(station_id):
"""
:type station_id: str
:rtype: dict[float, float]
"""
import os
import json
from pyticas_ncrtes import ncrtes
infra = ncrtes.get_infra()
json_path = os.path.join(infra.get_path('ncrtes', create=True), 'uk-snow',
'%s.json' % station_id)
if not os.path.exists(json_path):
return None
data = json.load(open(json_path, 'r'))
uk = {}
for k, u in data.items():
k = float(k)
while k in uk:
k += 0.000001
uk[k] = u
return uk
def prepare_data(tsi, station, year, stime, etime, snow_routes, reported, normal_func):
"""
:type tsi: pyticas_ncrtes.itypes.TargetStationInfo
:type station: pyticas.ttypes.RNodeObject
:type year: int
:type stime: datetime.datetime
:type etime: datetime.datetime
:type snow_routes: list[pyticas_ncrtes.core.etypes.SnowRoute]
:type reported: dict[str, list[pyticas_ncrtes.core.etypes.ReportedEvent]]
:type normal_months: list[(int, int)]
:rtype: pyticas_ncrtes.core.etypes.ESTData
"""
sevent = etypes.SnowEvent(stime, etime)
infra = ncrtes.get_infra()
target_station = infra.get_rnode(tsi.station_id)
tlcDA = TargetLaneConfigDataAccess()
tlci = tlcDA.get_by_station_id(year, station.station_id)
if tlci:
valid_detectors = [infra.get_detector(det_name.strip()) for det_name in tlci.detectors.split(',')]
else:
valid_detectors = target.get_target_detectors(station)
if not valid_detectors:
return None
try:
edata = etypes.ESTData(tsi, target_station, sevent, snow_routes, reported, normal_func)
edata.prepare_data(detectors=valid_detectors)
return edata
except Exception as ex:
log = getLogger(__name__)
log.warning('!!!!! Error : %s : %s' % (tsi.station_id, ex))
from pyticas.tool import tb
tb.traceback(ex)
return None
def _write(num, edata, output_path, prefix=''):
"""
:type num: int
:type edata: pyticas_ncrtes.core.etypes.ESTData
:type output_path: str
:type prefix: str
"""
logger = getLogger(__name__)
if edata is None or not edata.is_loaded:
logger.debug(' - Data is not loaded : %s' %
edata.target_station.station_id if edata else 'N/A')
return
sw = setting.SW_3HOURS
sks, sus = data_util.smooth(edata.ks, sw), data_util.smooth(edata.us, sw)
hills = hill_helper.create_uk_hills(edata.ks, edata.us, sks, sus)
ecs = ['#FF0000', '#FF7F00', '#FFFF00', '#00FF00', '#0000FF', '#9400D3']
long_title = '%s (%s[prj_id=%s, s_limit=%d, label=%s], %s)' % (
edata.snow_event.snow_period.get_date_string(),
edata.target_station.station_id, edata.snow_route.id
if edata.snow_route else 'N/A', edata.target_station.s_limit,
edata.target_station.label, edata.target_station.corridor.name)
title = '%s (%s, %s)' % (edata.snow_event.snow_period.get_date_string(),
edata.target_station.station_id,
edata.target_station.corridor.name)
if REPORT_MODE:
fig = plt.figure(figsize=(16, 9), dpi=100, facecolor='white')
else:
fig = plt.figure(dpi=100, facecolor='white')
ax1 = plt.subplot(211)
ax2 = plt.subplot(223)
ax3 = plt.subplot(224)
ax1.axhline(y=edata.target_station.s_limit, c='b')
if edata.wn_ffs:
ax1.axhline(y=edata.wn_ffs, c='r')
#ax1.plot(edata.us, marker='', label='Speed', c='#3794D5')
ax1.plot(edata.ks, marker='', label='Density', c='#ADE2CD')
# if edata.wn_avg_us is not None:
# ax1.plot(edata.wn_avg_us, c='#8C65C5', label='WN Avg. Speed')
# ax1.plot(edata.qus, c='#746F69', label='Q-Us')
if edata.normal_func:
nt_us = edata.normal_func.nighttime_func.speeds(
edata.snow_event.data_period.get_timeline(as_datetime=True))
else:
nt_us = [None] * edata.n_data
ax1.plot(nt_us, c='k', label='Nighttime Speed')
#nt_ks = edata.normal_func.nighttime_func.densities(edata.snow_event.data_period.get_timeline(as_datetime=True))
#ax1.plot(nt_ks, c='k', label='Nighttime Density')
# snow start and end time
sstime = edata.snow_event.time_to_index(edata.snow_event.snow_start_time)
setime = edata.snow_event.time_to_index(edata.snow_event.snow_end_time)
ax1.axvline(x=sstime, c='#948D90') # grey vertical line
ax1.axvline(x=setime, c='#948D90')
# draw chart using UK-Hills
n_data = len(edata.us)
ecs = ['#FF0000', '#FF7F00', '#FFFF00', '#00FF00', '#0000FF', '#9400D3']
markers = ['o', 'x', 'd', '^', '<', '>', 'v', 's', '*', '+']
ls = '-'
used = []
for gidx, hill in enumerate(hills):
lw = 1
c = ecs[gidx % len(ecs)]
marker = '|'
for midx in range(len(markers)):
marker_stick = '%s-%s' % (c, markers[midx])
if marker_stick not in used:
marker = markers[midx]
used.append(marker_stick)
break
sidx, eidx = hill.sidx, hill.eidx
_sidx, _eidx = sidx, eidx
ax2.plot(edata.sks[_sidx:_eidx + 1],
edata.sus[_sidx:_eidx + 1] * edata.sks[_sidx:_eidx + 1],
c=c,
marker=marker,
ms=4,
zorder=2)
ax3.plot(edata.sks[_sidx:_eidx + 1],
edata.sus[_sidx:_eidx + 1],
c=c,
marker=marker,
ms=4,
zorder=2)
_ks, _us = [None] * len(edata.ks), [None] * len(edata.us)
for idx in range(sidx, eidx + 1):
if idx >= n_data - 1:
break
# _ks[idx] = edata.merged_sks[idx]
# _us[idx] = edata.merged_sus[idx]
_ks[idx] = edata.ks[idx]
_us[idx] = edata.us[idx]
ax1.plot(_us, lw=lw, ls=ls, c=c)
# Normal and Wet-Normal Avg UK
_recv_ks = np.array(list(range(5, 120)))
if edata.normal_func:
uk_function = edata.normal_func.daytime_func.get_uk_function()
if uk_function and uk_function.is_valid():
ax2.plot(_recv_ks,
np.array(uk_function.speeds(_recv_ks)) * _recv_ks,
ls=':',
lw=2,
label='Normal Avg. QK')
# ax2.plot(_recv_ks, uk_function.speeds(_recv_ks), ls=':', lw=2, label='Normal Avg. UK')
ax3.plot(_recv_ks,
uk_function.speeds(_recv_ks),
ls=':',
lw=2,
label='Normal Avg. UK')
if uk_function._wn_uk:
ax2.plot(_recv_ks,
np.array(uk_function.wet_normal_speeds(_recv_ks)) *
_recv_ks,
ls=':',
lw=2,
label='Wet-Normal QK')
# ax2.plot(_recv_ks, uk_function.wet_normal_speeds(_recv_ks, edata.wn_ffs, edata.k_at_wn_ffs), ls=':', lw=2, label='Wet-Normal UK')
ax3.plot(_recv_ks,
uk_function.wet_normal_speeds(_recv_ks),
ls=':',
lw=2,
label='Wet-Normal UK')
# ax2.scatter(edata.ks, edata.us, c='#5D63FF', marker='.')
# ax3.scatter(edata.sks, edata.sus, c='#5D63FF', marker='.')
ms = [14, 13, 12, 11, 10, 8, 7]
us = data_util.smooth(edata.us, 15)
# srst
if edata.srst is not None:
data = np.array([None] * len(us))
data[edata.srst] = us[edata.srst]
ax1.plot(data, marker='o', ms=ms[0], c='#4ED8FF',
label='SRST') # light blue circle
# lst
if edata.lst is not None:
data = np.array([None] * len(us))
data[edata.lst] = us[edata.lst]
ax1.plot(data, marker='o', ms=ms[1], c='#FFAD2A',
label='LST') # orange circle
# sist
if edata.sist is not None:
data = np.array([None] * len(us))
data[edata.sist] = us[edata.sist]
ax1.plot(data, marker='o', ms=ms[2], c='#68BD20',
label='SIST') # orange green
# # rst
# if edata.rst is not None:
# data = np.array([None] * len(us))
# data[edata.rst] = us[edata.rst]
# ax1.plot(data, marker='o', ms=ms[2], c='#68BD20', label='RST') # green
# ncrt
if edata.ncrt is not None:
data = np.array([None] * len(us))
data[edata.ncrt] = us[edata.ncrt]
# ax1.plot(data, marker='o', ms=ms[3], c='#FF0512', label='NCRT (%d)' % edata.ncrt_type) # navy blue
ax1.plot(data, marker='o', ms=ms[3], c='#FF0512',
label='NCRT') # navy blue
# pst
if edata.pst:
data = np.array([None] * len(us))
data[edata.pst] = us[edata.pst]
ax1.plot(data, marker='o', ms=ms[4], c='#9535CC',
label='PST') # purple diamod
# sbpst
# if edata.sbpst:
# data = np.array([None] * len(us))
# data[edata.sbpst] = us[edata.sbpst]
# ax1.plot(data, marker='p', ms=ms[4], c='#bdb76b', label='BPST') # DarkKhaki
#
# # sapst
# if edata.sapst:
# data = np.array([None] * len(us))
# data[edata.sapst] = us[edata.sapst]
# ax1.plot(data, marker='p', ms=ms[4], c='#cdad00', label='APST') # gold3
# nfrt
if edata.nfrt:
data = np.array([None] * len(us))
data[edata.nfrt] = us[edata.nfrt]
ax1.plot(data, marker='*', ms=ms[4], c='#0072CC',
label='NFRT') # light blue
# # stable speed area around pst
# if not edata.ncrt and (edata.sbpst or edata.sapst):
# data = np.array([None] * len(us))
# if edata.sbpst:
# data[edata.sbpst] = us[edata.sbpst]
# ax1.plot(data, marker='p', ms=ms[4], c='#FF00F8', label='NCRT-C') # light-pink pentagon
# if edata.sapst and not edata.ncrt:
# data[edata.sapst] = us[edata.sapst]
# ax1.plot(data, marker='p', ms=ms[4], c='#FF00F8', label='') # light-pink pentagon
# snowday-ffs
# if edata.snowday_ffs:
# ax1.axhline(y=edata.snowday_ffs, c='#33CC0A')
# if edata.nfrt:
# data = np.array([None] * len(us))
# data[edata.nfrt] = us[edata.nfrt]
# ax1.plot(data, marker='*', ms=ms[5], c='#CC00B8', label='SnowDay-FFS') # navy blue
# reported time
for rp in edata.rps:
data = np.array([None] * len(us))
data[rp] = us[rp]
ax1.plot(data, marker='^', ms=ms[6], c='#FF0512',
label='RBRT') # red circle
box = ax1.get_position()
ax1.set_position([box.x0, box.y0, box.width * 0.9, box.height])
ax1.legend(loc='center left',
bbox_to_anchor=(1, 0.5),
numpoints=1,
prop={'size': 12})
ulist = np.array([90, 100, 110, 120, 150, 200, 300])
klist = np.array([80, 100, 150, 200, 250, 300, 350, 400, 450, 500, 600])
qlist = np.array([0, 2000, 3000, 3500, 4000, 4500, 5000, 6000, 7000])
_maxq = max(edata.sks * edata.sus)
_maxk = max(edata.sks)
_maxu = max(edata.sus)
maxq = qlist[np.where(qlist > _maxq)][0]
maxu = ulist[np.where(ulist > _maxu)][0]
maxk = klist[np.where(klist > _maxk)][0]
ax1.set_title(long_title)
ax2.set_title(title + ' - Smoothed Q-K')
ax3.set_title(title + ' - Smoothed U-K')
ax1.set_ylabel('speed, density')
ax1.set_xlabel('time')
ax1.set_ylim(ymin=0, ymax=maxu)
ax1.set_xlim(xmin=0, xmax=len(edata.sus))
# ax2.set_ylabel('speed')
ax2.set_ylabel('flow')
ax2.set_xlabel('density')
# ax2.set_ylim(ymin=0, ymax=100)
ax2.set_ylim(ymin=0, ymax=maxq)
ax2.set_xlim(xmin=0, xmax=maxk)
ax3.set_ylim(ymin=0, ymax=maxu)
ax3.set_xlim(xmin=0, xmax=maxk)
ax3.set_ylabel('speed')
ax3.set_xlabel('density')
ax1.grid()
ax2.grid()
ax3.grid()
ax2.legend(prop={'size': 12})
ax3.legend(prop={'size': 12})
if not REPORT_MODE:
plt.show()
else:
import datetime
timeline = edata.snow_event.data_period.get_timeline(as_datetime=True)
timeline = [
t - datetime.timedelta(seconds=setting.DATA_INTERVAL)
for t in timeline
]
ntimes = [
t.strftime('%H:%M') for idx, t in enumerate(timeline)
if idx % (2 * 3600 / setting.DATA_INTERVAL) == 0
]
loc_times = [
idx for idx, t in enumerate(timeline)
if idx % (2 * 3600 / setting.DATA_INTERVAL) == 0
]
ax1.set_xticks(loc_times)
ax1.set_xticklabels(ntimes, rotation=90)
plt.tight_layout()
postfix = ' (rp)' if edata.reported_events else ''
file_path = os.path.join(
output_path, '(%03d) %s%s%s.png' %
(num, prefix, edata.target_station.station_id, postfix))
fig.savefig(file_path, dpi=100, bbox_inches='tight')
plt.close(fig)
if 0:
wb = xlsxwriter.Workbook(
os.path.join(
ncrtes.get_infra().get_path('tmp', create=True),
'%s %s (night).xlsx' %
(edata.snow_event.snow_period.get_date_string(),
edata.target_station.station_id)))
ws = wb.add_worksheet('night-data')
prd = edata.snow_event.data_period.clone()
prd.interval = setting.DATA_INTERVAL
ws.write_column(0, 0, ['time'] + prd.get_timeline(as_datetime=False))
ws.write_column(0, 1, ['speed'] + edata.us.tolist())
ws.write_column(0, 2,
['nighttime average speed'] + edata.night_us.tolist())
ws.write_column(0, 3, ['smoothed speed'] + edata.sus.tolist())
# ws.write_column(0, 4, ['smoothed speed (2h)'] + edata.lsus.tolist())
# ws.write_column(0, 5, ['smoothed speed (1h)'] + edata.llsus.tolist())
ws.write_column(0, 6,
['nighttime ratio'] + edata.night_ratios.tolist())
wb.close()
def ncrtes_target_station_identification():
from pyticas_ncrtes import ncrtes
from pyticas_ncrtes.service import target_station_identification
from multiprocessing import Process
from pyticas import ticas
from pyticas_ncrtes.app import NCRTESApp
from pyticas_ncrtes.service.truck_route_updater import target_station_and_snowroute_info
logger = getLogger(__name__)
year = int(request.form.get('year'))
months = get_normal_months_from_year(year)
infra = ncrtes.get_infra()
ws = WinterSeasonDataAccess()
wsi = ws.get_by_months(months)
if not wsi:
wsi = itypes.WinterSeasonInfo()
wsi.set_months(months)
wsi.name = 'WinterSeason %s-%s' % (months[0][0], months[-1][0])
ws.insert(wsi, autocommit=True)
ws.close()
#######################
data_path = ticas._TICAS_.data_path
n_process = 5
stations = []
for n in range(n_process):
stations.append([])
for cidx, corr in enumerate(infra.get_corridors()):
if not corr.dir or corr.is_CD() or corr.is_Rev() or corr.is_RTMC():
continue
for sidx, st in enumerate(corr.stations):
if not st.is_normal_station() or not st.detectors:
continue
stations[sidx % n_process].append(st.station_id)
procs = []
for idx in range(n_process):
p = Process(target=target_station_identification.run,
args=(idx + 1, stations[idx], months, data_path,
NCRTESApp.DB_INFO))
p.start()
procs.append(p)
for p in procs:
p.join()
# print('# find alternatives....')
# for n in range(n_process):
# stations[n] = []
#
# for cidx, corr in enumerate(infra.get_corridors()):
# if not corr.dir or corr.is_CD() or corr.is_Rev() or corr.is_RTMC():
# continue
#
# counts = [ len(st_list) for idx, st_list in enumerate(stations)]
# tidx = counts.index(min(counts))
#
# for sidx, st in enumerate(corr.stations):
# if not st.is_normal_station() or not st.detectors:
# continue
# stations[tidx].append(st.station_id)
# for idx, stlist in enumerate(stations):
# print('# %d (%d): %s' % (idx, len(stlist), stlist))
# input('# enter to continue: ')
# procs = []
# for idx in range(n_process):
# p = Process(target=tsi_alternative.run, args=(idx + 1, stations[idx], months, data_path, NCRTESApp.DB_INFO))
# p.start()
# procs.append(p)
for p in procs:
p.join()
##################
# corr = infra.get_corridor_by_name('I-35W (NB)')
# for sidx, st in enumerate(corr.stations):
# nf = nsrf.get_normal_function(st, months)
target_station_and_snowroute_info(year)
return jsonify({'code': 1, 'message': 'success'})
def _prepare_est_data(stations, **kwargs):
"""
:type stations: list[pyticas_ncrtes.itypes.TargetStationInfo]
:type kwargs:
:rtype: (list[pyticas_ncrtes.core.etypes.ESTData], list[pyticas_ncrtes.itypes.TargetStationInfo], list[int])
"""
case_name = kwargs.get('case_name', None)
reported_events = kwargs.get('reported_events', None)
msg_queue = kwargs.get('msg_queue', None)
from_station = kwargs.get('from_station', None)
to_station = kwargs.get('to_station', None)
for_normalday = kwargs.get('for_normalday', None)
origin_prd = kwargs.get('origin_prd', None)
nd_offset = kwargs.get('nd_offset', None)
snow_routes = kwargs.get('snow_routes', None)
prd = kwargs.get('prd', None)
year = kwargs.get('year', None)
logger = getLogger(__name__)
infra = ncrtes.get_infra()
sidx_list, tsi_list, edata_list = [], [], []
msg_queue.put('Case : %s ' % case_name)
logger.debug(' > start : %s' % case_name)
# for test
f_found = False
f_to_found = False
# iterate target stations to prepare ESTData list
for sidx, tsi in enumerate(stations):
if f_to_found:
break
# for test
if from_station:
if tsi.station_id == from_station:
f_found = True
if not f_found:
continue
# target station object
station = infra.get_rnode(tsi.station_id)
if not station.is_normal_station() or not station.detectors:
continue
# for test
if to_station and tsi.station_id == to_station:
f_to_found = True
# for test
if for_normalday:
prd = normalday.get_normal_period(station, origin_prd, nd_offset)
reported_events = {}
msg_queue.put(' - Station : %s ' % tsi.station_id)
logger.debug(' - Station : %s' % tsi.station_id)
try:
# Normal Average UK Function
if tsi._normal_function:
nf = tsi._normal_function.func
else:
nf = None
edata = est.prepare_data(tsi, station, year, prd.start_date,
prd.end_date, snow_routes,
reported_events, nf)
except Exception as ex:
logger.debug(tb.traceback(ex, f_print=False))
continue
if not edata:
continue
edata_list.append(edata)
tsi_list.append(tsi)
sidx_list.append(sidx)
return edata_list, tsi_list, sidx_list
def run(pid, stations, months, data_path, db_info):
""" target station identification main process
Parameters
===========
- pid : process identification for multi-processing
- stations : station list
- months : month list
- data_path : TICAS data path
:type pid: int
:type stations: list[str]
:type months: list[(int, int)]
:type data_path : str
:type db_info: dict
:return:
"""
if db_info:
Infra.initialize(data_path)
infra = Infra.get_infra()
if conn.Session == None:
conn.connect(db_info)
else:
infra = ncrtes.get_infra()
logger = getLogger(__name__)
logger.info('starting target station identification')
wsDA = WinterSeasonDataAccess()
nfDA = NormalFunctionDataAccess()
tsDA = TargetStationDataAccess()
# process start time
stime = time.time()
n_stations = len(stations)
cnt = 0
for sidx, st in enumerate(stations):
station = infra.get_rnode(st)
logger.info('# PID=%d, SIDX=%d/%d, STATION=%s' % (pid, sidx, n_stations, st))
try:
nf = nsrf.get_normal_function(station, months, wsDA=wsDA, nfDA=nfDA, tsDA=tsDA, autocommit=True)
if nf and nf.is_valid():
logger.info(' - %s is valid' % station.station_id)
else:
logger.debug(' - %s is not valid (nf=%s)' % (station.station_id, nf))
# cnt += 1
#
# if cnt and cnt % 20 == 0:
# logger.warning(' - commmit!!')
# wsDA.commit()
# nfDA.commit()
# tsDA.commit()
except Exception as ex:
logger.warning(tb.traceback(ex, False))
# wsDA.commit()
# nfDA.commit()
# tsDA.commit()
# logger.warning(' - commmit!! (final)')
wsDA.close()
nfDA.close()
tsDA.close()
etime = time.time()
logger.info('end of target station identification (elapsed time=%s)' % timeutil.human_time(seconds=(etime - stime)))
def _chart(edata, ratios, lsratios, sratios):
"""
:type edata: pyticas_ncrtes.core.etypes.ESTData
:type ratios: numpy.ndarray
:type lsratios: numpy.ndarray
:type sratios: numpy.ndarray
"""
if not DEBUG_MODE:
return
import matplotlib.pyplot as plt
infra = ncrtes.get_infra()
output_path = infra.get_path('ncrtes/ut', create=True)
plt.figure(facecolor='white', figsize=(16, 8))
plt.plot(edata.lsks, edata.lsus)
_ks = list(range(5, 100))
_nus = edata.normal_func.daytime_func.recovery_speeds(_ks)
_wnus = edata.normal_func.daytime_func.recovery_function.wet_normal_speeds(_ks)
plt.plot(edata.lsks, edata.lsus)
plt.plot(_ks, _nus, c='k')
plt.plot(_ks, _wnus, c='orange')
plt.grid()
plt.figure(facecolor='white', figsize=(16, 8))
ax1 = plt.subplot(111)
ax1.plot(edata.us, c='#aaaaaa')
ax1.plot(edata.ks, c='#90A200')
ax1.plot(edata.night_us, c='k')
ax1.plot(edata.qus, c='r')
# ax1.plot(edata.qks, c='#C2CC34')
ax1.plot(edata.sus, c='#0B7CCC')
# ax1.plot(edata.qus, c='#CC50AA')
if edata.wn_ffs:
ax1.axhline(y=edata.wn_ffs, c='r', label='wn_ffs') # r
if edata.snowday_ffs:
ax1.axhline(y=edata.snowday_ffs, c='#E5E900', label='snowday_ffs') # light yellow
ax1.axhline(y=edata.target_station.s_limit, c='b')
if edata.ncrt_search_uth:
ax1.axhline(y=edata.ncrt_search_uth, c='k')
if edata.may_recovered_speed:
ax1.axhline(y=edata.may_recovered_speed, c='k')
ax1.axvline(x=edata.snow_event.snow_start_index, c='#EAEAEA', ls='-.')
ax1.axvline(x=edata.snow_event.snow_end_index, c='#EAEAEA', ls='-.')
if edata.ncrt_search_sidx:
ax1.axvline(x=edata.ncrt_search_sidx, c='b', ls='-.')
if edata.ncrt_search_eidx:
ax1.axvline(x=edata.ncrt_search_eidx, c='b', ls='-.')
if edata.wn_ffs_idx:
ax1.axvline(x=edata.wn_ffs_idx, c='g')
if edata.ncrt:
ax1.axvline(x=edata.ncrt, c='r', lw=2, label='ncrt')
ax2 = None
if ratios is not None and any(ratios):
ax2 = ax1.twinx()
# ax2.plot(edata.normal_ratios, c='#FF001A')
ax2.plot(lsratios, c='#00CC04', label='normal-ratios')
# ax2.plot(sratios, c='#CC34B1')
ax2.set_ylim(ymax=1.4, ymin=0.5)
if edata.wn_ffs and edata.wn_sratios is not None and any(edata.wn_sratios):
if not ax2:
ax2 = ax1.twinx()
ax2.plot(edata.wn_sratios, c='#CC07B7', label='wn-ratios')
ax2.plot(edata.wn_qratios, c='k', label='wn-qatios')
plt.title('%s (%s, s_limit=%d)' % (
edata.target_station.station_id,
edata.target_station.corridor.name,
edata.target_station.s_limit
))
plt.grid()
if ax2:
h1, l1 = ax1.get_legend_handles_labels()
h2, l2 = ax2.get_legend_handles_labels()
ax1.legend(h1+h2, l1+l2)
else:
ax1.legend()
plt.show()
filename = '%s.png' % edata.target_station.station_id
# plt.savefig(os.path.join(output_path, filename))
plt.show()
#
# filename = '%s.xlsx' % edata.target_station.station_id
# wb = xlsxwriter.Workbook(np.os.path.join(output_path, filename))
# ws = wb.add_worksheet('data')
# ws.write_column(0, 0, ['Time'] + edata.snow_event.data_period.get_timeline(as_datetime=False))
# ws.write_column(0, 1, ['Density'] + edata.ks.tolist() )
# ws.write_column(0, 2, ['Speed'] + edata.us.tolist() )
# ws.write_column(0, 3, ['QK'] + qks.tolist() )
# ws.write_column(0, 4, ['QU'] + qus.tolist() )
# if edata.normal_func and edata.normal_func.is_valid():
# klist = list(range(5, 100))
# ulist = edata.normal_func.daytime_func.recovery_speeds(klist)
# ws.write_column(0, 5, ['NormalK'] + klist )
# ws.write_column(0, 6, ['NormalU'] + ulist )
# wb.close()
def _write_chart_and_data(ddata, uk_origin, Kl, seg_func):
if not DEBUG_MODE:
return
ecs = ['#FF0000', '#FF7F00', '#FFFF00', '#00FF00', '#0000FF', '#9400D3']
cnt = 0
import matplotlib.pyplot as plt
plt.figure()
_us_origin, _ks_origin = data_util.dict2sorted_list(uk_origin)
ks_origin = np.array(_ks_origin)
us_origin = np.array(_us_origin)
plt.scatter(ks_origin, us_origin, marker='x', color='#999999')
for func in seg_func.funcs:
# print('> ', func.x1, func.x2, func.line_func, type(func))
isinstance(func, etypes.FitFunction)
uf = func.get_function()
lus = []
lks = []
for _k in range(int(func.x1 * 10), int(func.x2 * 10)):
k = _k / 10.0
if k > 140:
break
lks.append(k)
lus.append(uf(k))
c = ecs[cnt % len(ecs)]
cnt += 1
# plt.scatter(lks, lus, marker='^', color=c)
_ks, _us = [], []
for k in range(10, 200):
_ks.append(k)
_us.append(seg_func.speed(k))
plt.plot(_ks, data_util.smooth(_us, 11), c='b', lw=2)
plt.xlim(xmin=0, xmax=160)
plt.ylim(ymin=0, ymax=120)
plt.title('%s (Kt=%.2f, SL=%s, label=%s)' %
(ddata.station.station_id, Kl, ddata.station.s_limit,
ddata.station.label))
plt.grid()
periods = sorted(ddata.periods + ddata.not_congested_periods,
key=lambda prd: prd.start_date)
ffpath = _output_path(
'normal_function', '(%d-%d) %s.png' %
(periods[0].start_date.year, periods[-1].end_date.year,
ddata.station.station_id))
plt.tight_layout()
plt.savefig(ffpath, dpi=100, bbox_inches='tight')
# plt.show()
plt.close()
import xlsxwriter
import os
from pyticas_ncrtes import ncrtes
infra = ncrtes.get_infra()
output_dir = infra.get_path('ncrtes/normal-data-set', create=True)
data_file = os.path.join(
output_dir, '(%d-%d) %s (func).xlsx' %
(periods[0].start_date.year, periods[-1].end_date.year,
ddata.station.station_id))
wb = xlsxwriter.Workbook(data_file)
ws = wb.add_worksheet('uk')
ws.write_column(0, 0, ['k'] + _ks)
ws.write_column(0, 1, ['u'] + _us)
wb.close()
def ncrtes_ts_update():
year = int(request.form.get('year'))
station_id = request.form.get('station_id')
detectors = request.form.get('detectors')
infra = ncrtes.get_infra()
str_detectors = str(detectors)
if not year:
return prot.response_fail('Year Info must be entered.')
if not station_id:
return prot.response_fail('Station Info must be entered.')
if not detectors:
return prot.response_fail('Detectors must be entered.')
station = infra.get_rnode(station_id)
if not station:
return prot.response_fail('Station %s does not exists.' %
station_id)
station_detectors = [det.name for det in station.detectors]
detectors = [x.strip() for x in detectors.split(',')]
for det_name in detectors:
det = infra.get_detector(det_name)
if not det:
return prot.response_fail('Detector %s does not exists.' %
det_name)
if det.name not in station_detectors:
return prot.response_fail(
'Detector %s does not exists on the station.' % det_name)
wsDA = WinterSeasonDataAccess()
wsi = wsDA.get_by_year(year)
if not wsi:
months = get_normal_months_from_year(year)
wsi = itypes.WinterSeasonInfo()
wsi.set_months(months)
wsi.name = 'WinterSeason %s-%s' % (months[0][0], months[-1][0])
wsDA.insert(wsi, autocommit=True)
wsDA.close()
tsDA = TargetStationDataAccess()
ex_tsi = tsDA.get_by_station_id(year, station_id)
tsDA.close()
tlcDA = TargetLaneConfigDataAccess()
ex_tlci = tlcDA.get_by_station_id(year, station_id)
if not ex_tlci:
ex_tlci = itypes.TargetLaneConfigInfo()
ex_tlci.winterseason_id = wsi.id
ex_tlci.station_id = station_id
ex_tlci.detectors = ''
ex_tlci.corridor_name = station.corridor.name
model = tlcDA.insert(ex_tlci, autocommit=True)
ex_tlci.id = model.id
# update normal function
if ex_tsi and ex_tlci.detectors != str_detectors:
target_station = infra.get_rnode(ex_tlci.station_id)
valid_detectors = [
infra.get_detector(det_name) for det_name in detectors
]
normal_months = get_normal_months_from_year(year)
nf, nfi = None, None
try:
nf, nfi = _update_normal_function(target_station,
normal_months,
valid_detectors)
except Exception as ex:
from pyticas.tool import tb
tb.traceback(ex)
# query to update database
ex_tlci.detectors = ','.join(detectors)
updated = tlcDA.update(ex_tlci.id, ex_tlci.get_dict(), autocommit=True)
tlcDA.close()
if updated:
return prot.response_success(obj=ex_tlci.id)
else:
return prot.response_fail("fail to update (id={})".format(
ex_tlci.id))
def _est_groups(year,
request_param,
snow_routes,
snri_list,
from_station=None,
to_station=None):
"""
:type year: int
:type request_param: pyticas_ncrtes.itypes.EstimationRequestInfo
:type snow_routes: list[etypes.SnowRoute]
:type snri_list: list[pyticas_ncrtes.itypes.SnowRouteInfo]
:rtype: dict[str, list[pyticas_ncrtes.itypes.TargetStationInfo]]
"""
infra = ncrtes.get_infra()
logger = getLogger(__name__)
tsDA = TargetStationDataAccess()
nfDA = NormalFunctionDataAccess()
# make station groups by corridor or snow route
est_groups = {}
TS_CACHE = {}
if hasattr(request_param,
'target_stations') and request_param.target_stations:
target_stations = [
tsDA.get_by_station_id(year, st)
for st in request_param.target_stations
]
est_groups['user-defined'] = [tsi for tsi in target_stations if tsi]
elif request_param.target_corridors:
logger.debug('!estimate > prepare station list for each corridor')
for corr_name in request_param.target_corridors:
# TODO: for test
if 'I-94(EB)' == corr_name:
from_station = 'S1115'
to_station = 'S1051'
elif 'I-94(WB)' == corr_name:
from_station = 'S1058'
to_station = 'S1111'
logger.debug(' -> %s' % corr_name)
tsis = tsDA.list_by_corridor_name(year, corr_name)
if not tsis:
logger.debug('!!! no target stations for %s' % corr_name)
continue
# TODO: for test
f_found, f_to_found = False, False
_tsis = []
for tsi in tsis:
if f_to_found:
break
# for test
if from_station:
if tsi.station_id == from_station:
f_found = True
if not f_found:
continue
if to_station and tsi.station_id == to_station:
f_to_found = True
_tsis.append(tsi)
if from_station:
tsis = _tsis
###############
corr = infra.get_corridor_by_name(corr_name)
st_indices = {
st.station_id: idx
for idx, st in enumerate(corr.stations)
}
stsis = sorted(tsis, key=lambda tsi: st_indices[tsi.station_id])
if from_station:
sidx = st_indices.get(from_station, None)
eidx = st_indices.get(to_station, None)
if sidx and eidx:
stsis = [
tsi for tsi in stsis
if st_indices[tsi.station_id] >= sidx
or st_indices[tsi.station_id] <= sidx
]
# remove `station`s in curve
candidates = target.station.get_target_stations(corr_name)
candidate_stations = [st for st in candidates]
candidate_station_ids = [st.station_id for st in candidates]
_group = [
tsi for tsi in stsis if tsi.snowroute_name
and tsi.station_id in candidate_station_ids
]
_group_st_ids = [
tsi.station_id for tsi in _group if tsi._normal_function
]
res = []
for st in candidate_stations:
# no target station
if st.station_id not in _group_st_ids:
tsi = TargetStationInfo()
tsi.station_id = st.station_id
tsi.detectors = ','.join(
[det.name for det in target.get_target_detectors(st)])
snri = _find_snow_route(st, snri_list)
if snri:
tsi.snowroute_name = snri.name
tsi.snowroute_id = snri.id
nfi = nfDA.get_by_station(year, st.station_id)
if nfi:
tsi.normal_function_id = nfi.id
tsi._normal_function = nfi
res.append(tsi)
else:
# target station
_sts = [
tsi for tsi in stsis if tsi.station_id == st.station_id
]
res.append(_sts[0])
est_groups[corr_name] = res
elif request_param.target_snow_routes:
logger.debug('!estimate > prepare station list for each snow route')
for snow_route_id in request_param.target_snow_routes:
logger.debug(' -> snow_route.id=%d' % snow_route_id)
snri, target_stations1, target_stations2 = _get_stations_from_snowrouteinfos(
year, snow_route_id, snri_list, tsDA, TS_CACHE)
if not snri:
logger.warn('Cannot find snow-route information')
continue
if not target_stations1:
logger.debug(
'!!! no target stations for route1 of snowroute.id=%d' %
snow_route_id)
else:
est_groups['%d-route1' % snri.id] = target_stations1
if not target_stations2:
logger.debug(
'!!! no target stations for route2 of snowroute.id=%d' %
snow_route_id)
else:
est_groups['%d-route2' % snri.id] = target_stations2
tsDA.close()
# Remove the target stations having normal-function adjusted from other target station
# for k in est_groups.keys():
# est_groups[k] = [ tsi for tsi in est_groups[k] if not tsi._normal_function or not tsi._normal_function.adjusted_from]
for k in est_groups.keys():
est_groups[k] = [tsi for tsi in est_groups[k] if tsi.snowroute_id]
return est_groups
def estimate(request_param, msg_queue=None, **kwargs):
"""
:type request_param: pyticas_ncrtes.itypes.EstimationRequestInfo
:type msg_queue: queue.Queue
:rtype: list[pyticas_ncrtes.est.etypes.SnowData]
"""
logger = getLogger(__name__)
from_station = kwargs.get('from_station', None)
to_station = kwargs.get('to_station', None)
for_normalday = kwargs.get('normalday', None)
nd_offset = kwargs.get('nd_offset', 2)
stime = time.time()
infra = ncrtes.get_infra()
msg_queue = msg_queue or Queue()
logger.info('@@@ NCRT estimation process start')
# prepare reported events
reported_events = _reported_events(request_param)
# prepare time period and route information
year = _get_year(request_param.snow_start_time)
snow_routes, snri_list = _snow_routes(year)
prd = _snow_event(request_param)
origin_prd = prd.clone()
# make station groups by corridor or snow route
# to run by the the group
logger.info('year : %d' % year)
est_groups = _est_groups(year, request_param, snow_routes, snri_list,
from_station, to_station)
if not est_groups:
logger.info('! No target stations to estimate')
return False
# run estimation process for each station group
logger.debug('!estimate > run estimation process for each station group')
for case_name, stations in est_groups.items():
msg_queue.put('Case : %s ' % case_name)
logger.debug(' > start : %s' % case_name)
# iterate target stations to prepare ESTData list
edata_list, tsi_list, sidx_list = _prepare_est_data(
stations,
prd=prd,
year=year,
reported_events=reported_events,
snow_routes=snow_routes,
msg_queue=msg_queue,
from_station=from_station,
to_station=to_station,
for_normalday=for_normalday,
origin_prd=origin_prd,
nd_offset=nd_offset)
# logger.debug(' > finding NCRT by u')
# for edata in edata_list:
# ncrt_by_u.estimate(edata)
# logger.debug('end...')
# return
logger.debug(' > creating sections')
sections = section.sections(edata_list)
logger.debug(' > finding interested points')
# determine u-k change points by snowing
# bysnowing.find(edata_list, sections)
logger.debug(' > start estimating')
for idx, edata in enumerate(edata_list):
# Run estimation process
# logger.debug(' ==> %s' % edata.target_station.station_id)
est.estimate(sidx_list[idx], tsi_list[idx], edata)
# adjust NCRT of individual stations
logger.debug(' > sectionwide adjust NCRTs')
# TODO: enable this...
ncrt_sectionwide.adjust_ncrts(edata_list, sections)
# remove single-station section without NCRT (type1)
edata_list, sections = ncrt_sectionwide.remove_section_with_single_station(
edata_list, sections)
# determine NCRT (type2) using section-wide data
# logger.debug(' > sectionwide alternatives')
# ncrt_sectionwide.determine(edata_list, sections)
# determine phase change points
logger.debug(' > phase change points')
for idx, edata in enumerate(edata_list):
try:
phase.determine(edata)
except Exception as ex:
tb.traceback(ex)
continue
output_path = os.path.join(infra.get_path('ncrtes',
create=True), 'whole_data',
prd.start_date.strftime('%Y-%m-%d'),
case_name)
est.report(case_name, edata_list, output_path)
etime = time.time()
logger.info('@@@ End of NCRT estimation process (elapsed time=%s)' %
timeutil.human_time(seconds=(etime - stime)))
def dumps_function_without_commit(ncr_func,
months,
wsDA,
nfDA,
tsDA,
autocommit=False):
"""
:type ncr_func: pyticas_ncrtes.core.etypes.NSRFunction
:type months: list[(int, int)]
:type wsDA: WinterSeasonDataAccess
:type nfDA: NormalFunctionDataAccess
:type tsDA: TargetStationDataAccess
:rtype: itypes.NormalFunctionInfo
"""
wsi = wsDA.get_by_months(months)
if not wsi:
wsi = itypes.WinterSeasonInfo()
wsi.set_months(months)
wsi.name = 'WinterSeason %s-%s' % (months[0][0], months[-1][0])
wsDA.insert(wsi, autocommit=True)
nfi = itypes.NormalFunctionInfo()
nfi.func = ncr_func
nfi.station_id = ncr_func.station.station_id
nfi.winterseason_id = wsi.id
ex = nfDA.get_by_station(wsi.id, ncr_func.station.station_id)
if ex:
nfDA.delete(ex.id, autocommit=True)
nfDA.insert(nfi, autocommit=autocommit)
if ncr_func.is_valid():
tsi = itypes.TargetStationInfo()
tsi.station_id = ncr_func.station.station_id
tsi.winterseason_id = wsi.id
tsi.corridor_name = ncr_func.station.corridor.name
tsi.sroute_id = None
tsi.normal_function_id = nfi.id
infra = ncrtes.get_infra()
target_dets = get_target_detectors(infra.get_rnode(tsi.station_id))
ex = tsDA.get_by_station_id(wsi.year, tsi.station_id, as_model=True)
if ex:
if not target_dets:
ex.detectors = None
else:
ex.detectors = ','.join([det.name for det in target_dets])
if autocommit:
tsDA.commit()
else:
if not target_dets:
tsi.detectors = None
else:
tsi.detectors = ','.join([det.name for det in target_dets])
tsDA.insert(tsi)
if autocommit:
tsDA.commit()
return nfi
def collect(target_station, periods, **kwargs):
""" return recovery and reduction pattern UK data set
:type target_station: pyticas.ttypes.RNodeObject
:type periods: list[pyticas.ttypes.Period]
:rtype: (list[pyticas.ttypes.Period],
dict[float, float], dict[float, float], dict[float, float],
list[list[float]], list[list[float]],
list[pyticas.ttypes.RNodeData], list[pyticas.ttypes.RNodeData],
list[pyticas.ttypes.DetectorObject,
list[pyticas.ttypes.Period],
dict[float, float])
"""
# Procedure
# 1. check malfunctioned detector in a given target station
# 2. iterate for all time periods
# 2.1 call _collect_data_a_data with a time period
# 2.2 save the daily data from 2.1
global wb
logger = getLogger(__name__)
allow_not_congested = kwargs.get('allow_not_congested', False)
dc, valid_detectors = kwargs.get('dc', None), kwargs.get('valid_detectors', None)
if not dc:
dc, valid_detectors = lane.get_detector_checker(target_station)
if not dc:
return None, None, None, None, None, None, None, None, None, None, None
n_count = 0
uss, kss, rnode_data_ks, rnode_data_us, used_periods, all_hills = ([] for _ in range(6))
all_patterns, recovery_patterns, reduction_patterns = {}, {}, {}
import xlsxwriter
import os
from pyticas_ncrtes import ncrtes
infra = ncrtes.get_infra()
output_dir = infra.get_path('ncrtes/normal-data-set', create=True)
data_file = os.path.join(output_dir, '(%d-%d) %s.xlsx' % (periods[0].start_date.year, periods[-1].end_date.year, target_station.station_id))
wb = xlsxwriter.Workbook(data_file)
not_congested_periods, not_congested_patterns = [], {}
for prd in periods:
all_uk, recovery_uk, reduction_uk, u_all, k_all, us, ks, hills_a_day, is_not_congested = _collect_data_a_day(target_station, prd, dc)
if not recovery_uk:
continue
if is_not_congested:
not_congested_periods.append(prd)
for rk in recovery_uk.keys():
_tk = rk if rk in not_congested_patterns else _unique_key(rk, list(not_congested_patterns.keys()))
if not _tk:
continue
not_congested_patterns[_tk] = recovery_uk[rk]
continue
if recovery_uk and reduction_uk:
uss.append(u_all)
kss.append(k_all)
rnode_data_us.append(us)
rnode_data_ks.append(ks)
n_count += 1
for rk in recovery_uk.keys():
_tk = rk if rk in recovery_patterns else _unique_key(rk, list(recovery_patterns.keys()))
if not _tk:
continue
recovery_patterns[_tk] = recovery_uk[rk]
used_periods.append(prd)
ks = list(recovery_patterns.keys())
us = [ recovery_patterns[_k] for _k in ks ]
if has_abnormal_ffs(target_station, recovery_patterns):
logger.debug('!! has abnormal ffs : station=%s, s_limit=%s' % (target_station.station_id, target_station.s_limit))
all_uk = dict(not_congested_patterns)
all_uk.update(recovery_patterns)
ks = list(all_uk.keys())
us = [ all_uk[_k] for _k in ks ]
ws = wb.add_worksheet('uk')
ws.write_column(0, 0, ['k'] + ks)
ws.write_column(0, 1, ['u'] + us)
wb.close()
return None, None, None, None, None, None, None, None, None, None, None
ks, us = filter_abnormal_data(target_station, ks, us)
recovery_patterns = { _k : us[idx] for idx, _k in enumerate(ks) }
reduction_patterns = recovery_patterns
all_patterns = recovery_patterns
ws = wb.add_worksheet('uk')
ws.write_column(0, 0, ['k'] + ks)
ws.write_column(0, 1, ['u'] + us)
wb.close()
return (used_periods, all_patterns, recovery_patterns, reduction_patterns,
uss, kss, rnode_data_us, rnode_data_ks, valid_detectors,
not_congested_periods, not_congested_patterns)
