def _handler_ttroute(da, item, action_log):
"""
:type da: pyticas_tetres.da.route.TTRouteDataAccess
:type item: pyticas_tetres.ttypes.TTRouteInfo
:type action_log: pyticas_tetres.ttypes.ActionLogInfo
"""
# 1. calculate travel time
# 2. categorize (all)
try:
from pyticas_tetres.util.traffic_file_checker import has_traffic_files
start = datetime.date(cfg.DATA_ARCHIVE_START_YEAR, 1, 1)
last_day = datetime.date.today() - datetime.timedelta(days=cfg.DAILY_JOB_OFFSET_DAYS)
start_date_str, end_date_str = start.strftime('%Y-%m-%d'), last_day.strftime('%Y-%m-%d')
if not has_traffic_files(start_date_str, end_date_str):
return False, "Missing traffic files for the given time range from {} to {}.".format(start_date_str, end_date_str)
except Exception as e:
getLogger(__name__).warning(
'Exception occured while checking if traffic files exist during handling travel time routes. Error: {}'.format(
e))
daily_periods = _get_all_daily_periods()
cnt = 0
for prd in daily_periods:
try:
inserted_ids = traveltime.calculate_a_route(prd, item)
if inserted_ids:
categorization.categorize(item, prd)
cnt += len(inserted_ids)
except Exception as ex:
getLogger(__name__).warning(
'Exception occured when handling route changes : %s' % tb.traceback(ex, f_print=False))
return cnt > 0
def _handler_ttroute(da, item, action_log):
"""
:type da: pyticas_tetres.da.route.TTRouteDataAccess
:type item: pyticas_tetres.ttypes.TTRouteInfo
:type action_log: pyticas_tetres.ttypes.ActionLogInfo
"""
# 1. calculate travel time
# 2. categorize (all)
# faverolles 1/16/2020 NOTE: always starts at datetime.today
daily_periods = _get_all_daily_periods()
cnt = 0
try:
for prd in daily_periods:
inserted_ids = traveltime.calculate_a_route(prd, item)
if inserted_ids:
categorization.categorize(item, prd)
if inserted_ids is not False:
cnt += len(inserted_ids)
return cnt > 0
except Exception as ex:
getLogger(__name__).warning(
'Exception occured when handling route changes : %s' %
tb.traceback(ex, f_print=False))
return False
def _perform_calculation_of_tt(ttri, prd):
"""
:type ttri: pyticas_tetres.ttypes.TTRouteInfo
:type prd: pyticas.ttypes.Period
"""
inserted_ids = traveltime.calculate_a_route(prd, ttri)
if inserted_ids:
categorization.categorize(ttri, prd)
def _categorize_for_a_day(prd, categorizer, **kwargs):
"""
:type prd: pyticas.ttypes.Period
:type categorizer: any
"""
da_route = route.TTRouteDataAccess()
route_list = da_route.list()
for a_route in route_list:
categorization.categorize(a_route, prd, categorizers=[categorizer], **kwargs)
da_route.close_session()
def run(prd):
"""
:type prd: pyticas.ttypes.Period
:return:
"""
tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365)
ttr_route_da = TTRouteDataAccess()
routes = ttr_route_da.list()
ttr_route_da.close_session()
logger = getLogger(__name__)
has_error = 0
for ttri in routes:
try:
result = DataCategorizer.categorize(ttri, prd)
if result['has_error']:
logger.debug(
' - error occured when doing categorization for route %s (id=%s) during %s'
% (ttri.name, ttri.id, prd.get_date_string()))
tlogger.add_log({
'time': tlogger.now(),
'route_id': ttri.id,
'target_period': prd,
'failed': True
})
has_error += 1
except Exception as ex:
logger.debug(
' - exception occured when doing categorization for route %s (id=%s) during %s'
% (ttri.name, ttri.id, prd.get_date_string()))
tlogger.add_log({
'time': tlogger.now(),
'route_id': ttri.id,
'target_period': prd,
'failed': True
})
has_error += 1
logger.debug(' - categorization for %s routes are done (has_error=%s)' %
(len(routes), has_error))
tlogger.set_registry('last_executed', tlogger.now())
tlogger.save()
def _handler_systemconfig(da, item, action_log):
"""
:type da: pyticas_tetres.da.config.ConfigDataAccess
:type item: pyticas_tetres.ttypes.SystemConfigInfo
:type action_log: pyticas_tetres.ttypes.ActionLogInfo
"""
ttr_da = TTRouteDataAccess()
routes = ttr_da.list()
ttr_da.close_session()
start_date = datetime.datetime.strptime('%s-01-01' % cfg.DATA_ARCHIVE_START_YEAR, '%Y-%m-%d')
last_date = datetime.datetime.now() - datetime.timedelta(days=cfg.DAILY_JOB_OFFSET_DAYS)
daily_periods = _get_daily_periods(start_date, last_date)
# faverolles 1/16/2020 NOTE: If the AdminClient changes the Archive Start Year parameter
# and an entry is made in the action_log database table.
# The server will repeatedly rerun initial_data_maker which is also run by (dataloader.py)
# until this entry is removed from the database.
# The issue is that the entry won't be removed because "target data is not handled" which
# i think means "until all traffic data is downloaded" for the archive start year.
# This never happens because the traffic data is hundreds of GB's.
#
if action_log.data_desc.startswith('DATA_ARCHIVE_START_YEAR_EXTENDED'):
# calculate travel time data and the related non-traffic data during the extended years
_, year_change = action_log.data_desc.split(':')
prev_year, changed_year = year_change.split('->')
prev_year, changed_year = int(prev_year.strip()), int(changed_year.strip())
prev_end_date = datetime.datetime.strptime('%s-12-31' % (prev_year - 1), '%Y-%m-%d').date()
try:
# faverolles 1/16/2020 NOTE: Why is there no parameter db_info passed
# I'm guessing its expected to fail because try-catch maybe?
from pyticas_tetres.util.traffic_file_checker import has_traffic_files
start_date_str, end_date_str = start_date.strftime('%Y-%m-%d'), prev_end_date.strftime('%Y-%m-%d')
if not has_traffic_files(start_date_str, end_date_str):
return False, "Missing traffic files for the given time range from {} to {}.".format(start_date_str, end_date_str)
import dbinfo
initial_data_maker.run(start_date.date(), prev_end_date, db_info=dbinfo.tetres_db_info())
return True
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - Data Archive Start Year (Extended) : %s'
% tb.traceback(ex, f_print=False))
return False
elif action_log.data_desc.startswith('DATA_ARCHIVE_START_YEAR_SHRINKED'):
# delete the travel time data and the related non-traffic data during the shrinked years
_, year_change = action_log.data_desc.split(':')
prev_year, changed_year = year_change.split('->')
prev_year, changed_year = int(prev_year.strip()), int(changed_year.strip())
years = [y for y in range(prev_year, changed_year)]
for y in years:
sdt = datetime.datetime.strptime('%s-01-01 00:00:00' % y, '%Y-%m-%d %H:%M:%S')
edt = datetime.datetime.strptime('%s-12-31 23:59:59' % y, '%Y-%m-%d %H:%M:%S')
try:
tt_da = TravelTimeDataAccess(y)
for a_route in routes:
tt_da.delete_range(a_route.id, sdt, edt)
tt_da.close_session()
weather_da = NoaaWeatherDataAccess(y)
weather_da.delete_range(None, None, start_time=sdt, end_time=edt)
weather_da.commit()
weather_da.close_session()
incident_da = IncidentDataAccess()
incident_da.delete_range_all(start_time=sdt, end_time=edt)
incident_da.commit()
incident_da.close_session()
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - Data Archive Start Year (Shrinked) : %s'
% tb.traceback(ex, f_print=False))
return False
elif action_log.target_datatype == ActionLogDataAccess.DT_INCIDENT:
for a_route in routes:
for prd in daily_periods:
try:
categorization.categorize(a_route, prd, categorizers=[categorization.incident])
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - Incident Parameters Changes : %s'
% tb.traceback(ex, f_print=False))
return False
elif action_log.target_datatype == ActionLogDataAccess.DT_WORKZONE:
for a_route in routes:
for prd in daily_periods:
try:
categorization.categorize(a_route, prd, categorizers=[categorization.workzone])
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - Workzone Parameters Changes : %s'
% tb.traceback(ex, f_print=False))
return False
elif action_log.target_datatype == ActionLogDataAccess.DT_SPECIALEVENT:
for a_route in routes:
for prd in daily_periods:
try:
categorization.categorize(a_route, prd, categorizers=[categorization.specialevent])
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - SpecialEvent Parameters Changes : %s'
% tb.traceback(ex, f_print=False))
return False
elif action_log.target_datatype == ActionLogDataAccess.DT_SNOWMGMT:
for a_route in routes:
for prd in daily_periods:
try:
categorization.categorize(a_route, prd, categorizers=[categorization.snowmgmt])
except Exception as ex:
getLogger(__name__).warning(
'exception occured when handling SystemConfig - SnowManagement Parameters Changes : %s'
% tb.traceback(ex, f_print=False))
return False
return True