def _handler_ttroute(da, item, action_log): """ :type da: pyticas_tetres.da.route.TTRouteDataAccess :type item: pyticas_tetres.ttypes.TTRouteInfo :type action_log: pyticas_tetres.ttypes.ActionLogInfo """ # 1. calculate travel time # 2. categorize (all) try: from pyticas_tetres.util.traffic_file_checker import has_traffic_files start = datetime.date(cfg.DATA_ARCHIVE_START_YEAR, 1, 1) last_day = datetime.date.today() - datetime.timedelta(days=cfg.DAILY_JOB_OFFSET_DAYS) start_date_str, end_date_str = start.strftime('%Y-%m-%d'), last_day.strftime('%Y-%m-%d') if not has_traffic_files(start_date_str, end_date_str): return False, "Missing traffic files for the given time range from {} to {}.".format(start_date_str, end_date_str) except Exception as e: getLogger(__name__).warning( 'Exception occured while checking if traffic files exist during handling travel time routes. Error: {}'.format( e)) daily_periods = _get_all_daily_periods() cnt = 0 for prd in daily_periods: try: inserted_ids = traveltime.calculate_a_route(prd, item) if inserted_ids: categorization.categorize(item, prd) cnt += len(inserted_ids) except Exception as ex: getLogger(__name__).warning( 'Exception occured when handling route changes : %s' % tb.traceback(ex, f_print=False)) return cnt > 0
def _handler_ttroute(da, item, action_log): """ :type da: pyticas_tetres.da.route.TTRouteDataAccess :type item: pyticas_tetres.ttypes.TTRouteInfo :type action_log: pyticas_tetres.ttypes.ActionLogInfo """ # 1. calculate travel time # 2. categorize (all) # faverolles 1/16/2020 NOTE: always starts at datetime.today daily_periods = _get_all_daily_periods() cnt = 0 try: for prd in daily_periods: inserted_ids = traveltime.calculate_a_route(prd, item) if inserted_ids: categorization.categorize(item, prd) if inserted_ids is not False: cnt += len(inserted_ids) return cnt > 0 except Exception as ex: getLogger(__name__).warning( 'Exception occured when handling route changes : %s' % tb.traceback(ex, f_print=False)) return False
def _perform_calculation_of_tt(ttri, prd): """ :type ttri: pyticas_tetres.ttypes.TTRouteInfo :type prd: pyticas.ttypes.Period """ inserted_ids = traveltime.calculate_a_route(prd, ttri) if inserted_ids: categorization.categorize(ttri, prd)
def _categorize_for_a_day(prd, categorizer, **kwargs): """ :type prd: pyticas.ttypes.Period :type categorizer: any """ da_route = route.TTRouteDataAccess() route_list = da_route.list() for a_route in route_list: categorization.categorize(a_route, prd, categorizers=[categorizer], **kwargs) da_route.close_session()
def run(prd): """ :type prd: pyticas.ttypes.Period :return: """ tlogger = task_logger.get_task_logger(TASK_LOGGER_NAME, capacity=365) ttr_route_da = TTRouteDataAccess() routes = ttr_route_da.list() ttr_route_da.close_session() logger = getLogger(__name__) has_error = 0 for ttri in routes: try: result = DataCategorizer.categorize(ttri, prd) if result['has_error']: logger.debug( ' - error occured when doing categorization for route %s (id=%s) during %s' % (ttri.name, ttri.id, prd.get_date_string())) tlogger.add_log({ 'time': tlogger.now(), 'route_id': ttri.id, 'target_period': prd, 'failed': True }) has_error += 1 except Exception as ex: logger.debug( ' - exception occured when doing categorization for route %s (id=%s) during %s' % (ttri.name, ttri.id, prd.get_date_string())) tlogger.add_log({ 'time': tlogger.now(), 'route_id': ttri.id, 'target_period': prd, 'failed': True }) has_error += 1 logger.debug(' - categorization for %s routes are done (has_error=%s)' % (len(routes), has_error)) tlogger.set_registry('last_executed', tlogger.now()) tlogger.save()
def _handler_systemconfig(da, item, action_log): """ :type da: pyticas_tetres.da.config.ConfigDataAccess :type item: pyticas_tetres.ttypes.SystemConfigInfo :type action_log: pyticas_tetres.ttypes.ActionLogInfo """ ttr_da = TTRouteDataAccess() routes = ttr_da.list() ttr_da.close_session() start_date = datetime.datetime.strptime('%s-01-01' % cfg.DATA_ARCHIVE_START_YEAR, '%Y-%m-%d') last_date = datetime.datetime.now() - datetime.timedelta(days=cfg.DAILY_JOB_OFFSET_DAYS) daily_periods = _get_daily_periods(start_date, last_date) # faverolles 1/16/2020 NOTE: If the AdminClient changes the Archive Start Year parameter # and an entry is made in the action_log database table. # The server will repeatedly rerun initial_data_maker which is also run by (dataloader.py) # until this entry is removed from the database. # The issue is that the entry won't be removed because "target data is not handled" which # i think means "until all traffic data is downloaded" for the archive start year. # This never happens because the traffic data is hundreds of GB's. # if action_log.data_desc.startswith('DATA_ARCHIVE_START_YEAR_EXTENDED'): # calculate travel time data and the related non-traffic data during the extended years _, year_change = action_log.data_desc.split(':') prev_year, changed_year = year_change.split('->') prev_year, changed_year = int(prev_year.strip()), int(changed_year.strip()) prev_end_date = datetime.datetime.strptime('%s-12-31' % (prev_year - 1), '%Y-%m-%d').date() try: # faverolles 1/16/2020 NOTE: Why is there no parameter db_info passed # I'm guessing its expected to fail because try-catch maybe? from pyticas_tetres.util.traffic_file_checker import has_traffic_files start_date_str, end_date_str = start_date.strftime('%Y-%m-%d'), prev_end_date.strftime('%Y-%m-%d') if not has_traffic_files(start_date_str, end_date_str): return False, "Missing traffic files for the given time range from {} to {}.".format(start_date_str, end_date_str) import dbinfo initial_data_maker.run(start_date.date(), prev_end_date, db_info=dbinfo.tetres_db_info()) return True except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - Data Archive Start Year (Extended) : %s' % tb.traceback(ex, f_print=False)) return False elif action_log.data_desc.startswith('DATA_ARCHIVE_START_YEAR_SHRINKED'): # delete the travel time data and the related non-traffic data during the shrinked years _, year_change = action_log.data_desc.split(':') prev_year, changed_year = year_change.split('->') prev_year, changed_year = int(prev_year.strip()), int(changed_year.strip()) years = [y for y in range(prev_year, changed_year)] for y in years: sdt = datetime.datetime.strptime('%s-01-01 00:00:00' % y, '%Y-%m-%d %H:%M:%S') edt = datetime.datetime.strptime('%s-12-31 23:59:59' % y, '%Y-%m-%d %H:%M:%S') try: tt_da = TravelTimeDataAccess(y) for a_route in routes: tt_da.delete_range(a_route.id, sdt, edt) tt_da.close_session() weather_da = NoaaWeatherDataAccess(y) weather_da.delete_range(None, None, start_time=sdt, end_time=edt) weather_da.commit() weather_da.close_session() incident_da = IncidentDataAccess() incident_da.delete_range_all(start_time=sdt, end_time=edt) incident_da.commit() incident_da.close_session() except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - Data Archive Start Year (Shrinked) : %s' % tb.traceback(ex, f_print=False)) return False elif action_log.target_datatype == ActionLogDataAccess.DT_INCIDENT: for a_route in routes: for prd in daily_periods: try: categorization.categorize(a_route, prd, categorizers=[categorization.incident]) except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - Incident Parameters Changes : %s' % tb.traceback(ex, f_print=False)) return False elif action_log.target_datatype == ActionLogDataAccess.DT_WORKZONE: for a_route in routes: for prd in daily_periods: try: categorization.categorize(a_route, prd, categorizers=[categorization.workzone]) except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - Workzone Parameters Changes : %s' % tb.traceback(ex, f_print=False)) return False elif action_log.target_datatype == ActionLogDataAccess.DT_SPECIALEVENT: for a_route in routes: for prd in daily_periods: try: categorization.categorize(a_route, prd, categorizers=[categorization.specialevent]) except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - SpecialEvent Parameters Changes : %s' % tb.traceback(ex, f_print=False)) return False elif action_log.target_datatype == ActionLogDataAccess.DT_SNOWMGMT: for a_route in routes: for prd in daily_periods: try: categorization.categorize(a_route, prd, categorizers=[categorization.snowmgmt]) except Exception as ex: getLogger(__name__).warning( 'exception occured when handling SystemConfig - SnowManagement Parameters Changes : %s' % tb.traceback(ex, f_print=False)) return False return True