def _check_logs():
"""
:rtype: list[(int, pyticas.ttypes.Period)]
"""
if '_01_tt' not in sys.modules:
from pyticas_tetres.sched.daily_tasks import _01_tt, _02_load_weather, _03_load_incident, _04_tagging
non_completed_dates = []
# tt
tlogger = task_logger.get_task_logger(_01_tt.TASK_LOGGER_NAME,
capacity=365)
for idx, log in enumerate(tlogger.logs):
for vidx, ttri_id in enumerate(tlogger.logs[idx]['failed']):
non_completed_dates.append((ttri_id, log['target_period']))
tlogger.logs[idx]['failed'] = []
tlogger.save()
# weather (load if there is not-loaded weather data)
tlogger = task_logger.get_task_logger(_02_load_weather.TASK_LOGGER_NAME,
capacity=365)
for idx, log in enumerate(tlogger.logs):
for vidx, v in enumerate(tlogger.logs[idx]['failed']):
res = noaa_loader.import_daily_data_for_a_station(
log['target_period'].start_date, v['usaf'], v['wban'])
if res['loaded']:
del tlogger.logs[idx]['failed'][vidx]
tlogger.save()
# incident (load if there is not-loaded incident data)
tlogger = task_logger.get_task_logger(_03_load_incident.TASK_LOGGER_NAME,
capacity=365)
for idx, log in enumerate(tlogger.logs):
if not log['failed']:
continue
for vidx, v in enumerate(tlogger.logs[idx]['failed']):
n_inserted, has_error = incident_loader.import_all_corridor(
log['target_period'].start_date, log['target_period'].end_date)
if not has_error:
del tlogger.logs[idx]['failed'][vidx]
tlogger.save()
# tagging (load if there is not-loaded incident data)
tlogger = task_logger.get_task_logger(_04_tagging.TASK_LOGGER_NAME,
capacity=365)
for idx, log in enumerate(tlogger.logs):
if not log['failed']:
continue
non_completed_dates.append((log['route_id'], log['target_period']))
tlogger.logs[idx]['failed'] = []
tlogger.save()
return non_completed_dates
def run(prd):
"""
:type prd: pyticas.ttypes.Period
:return:
"""
tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365)
n_inserted, has_error = incident_loader.import_all_corridor(
prd.start_date, prd.end_date)
getLogger(__name__).debug(' - %s incidents are loaded (has_error=%s)' %
(n_inserted, has_error))
tlogger.set_registry('last_executed', tlogger.now())
tlogger.add_log({
'time': tlogger.now(),
'target_period': prd,
'failed': has_error
})
tlogger.save()
def run(prd):
"""
:type prd: pyticas.ttypes.Period
:return:
"""
tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365)
ttr_route_da = TTRouteDataAccess()
routes = ttr_route_da.list()
ttr_route_da.close_session()
logger = getLogger(__name__)
has_error = 0
for ttri in routes:
try:
result = DataCategorizer.categorize(ttri, prd)
if result['has_error']:
logger.debug(
' - error occured when doing categorization for route %s (id=%s) during %s'
% (ttri.name, ttri.id, prd.get_date_string()))
tlogger.add_log({
'time': tlogger.now(),
'route_id': ttri.id,
'target_period': prd,
'failed': True
})
has_error += 1
except Exception as ex:
logger.debug(
' - exception occured when doing categorization for route %s (id=%s) during %s'
% (ttri.name, ttri.id, prd.get_date_string()))
tlogger.add_log({
'time': tlogger.now(),
'route_id': ttri.id,
'target_period': prd,
'failed': True
})
has_error += 1
logger.debug(' - categorization for %s routes are done (has_error=%s)' %
(len(routes), has_error))
tlogger.set_registry('last_executed', tlogger.now())
tlogger.save()
def run(prd):
"""
:type prd: pyticas.ttypes.Period
:return:
"""
tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365)
res = traveltime.calculate_all_routes(prd)
failed = [v['route_id'] for v in res if not v['done']]
getLogger(__name__).debug(
' - travel times for %s routes are calculated (has_error=%s)' %
(len(res), len(failed)))
tlogger.set_registry('last_executed', tlogger.now())
tlogger.add_log({
'time': tlogger.now(),
'target_period': prd,
'failed': failed
})
tlogger.save()
def run(prd):
"""
:type prd: pyticas.ttypes.Period
:return:
"""
tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365)
res = noaa_loader.import_daily_data(prd.start_date)
logger = getLogger(__name__)
for v in res:
logger.debug(' - weather station(usaf=%s, wban=%s) : %d data loaded' %
(v['usaf'], v['wban'], v['loaded']))
failed = [v for v in res if not v['loaded']]
tlogger.set_registry('last_executed', tlogger.now())
tlogger.add_log({
'time': tlogger.now(),
'target_period': prd,
'failed': failed
})
tlogger.save()