def initialize_system_config_info():
da = ConfigDataAccess()
config_data = da.get_by_name(cfg.OPT_NAME_SYSCONFIG)
if not config_data or not config_data.content:
syscfg = get_system_config_info()
da.insert_or_update(cfg.OPT_NAME_SYSCONFIG, json.dumps(syscfg))
da.commit()
else:
syscfg = json.loads(config_data.content)
cfg.DATA_ARCHIVE_START_YEAR = syscfg.data_archive_start_year
cfg.DAILY_JOB_OFFSET_DAYS = syscfg.daily_job_offset_days
cfg.DAILY_JOB_START_TIME = syscfg.daily_job_start_time
cfg.WEEKLY_JOB_START_WEEKDAY = syscfg.weekly_job_start_day
cfg.WEEKLY_JOB_START_TIME = syscfg.weekly_job_start_time
cfg.MONTHLY_JOB_START_DAY = syscfg.monthly_job_start_date
cfg.MONTHLY_JOB_START_TIME = syscfg.monthly_job_start_time
cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT = syscfg.incident_downstream_distance_limit
cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT = syscfg.incident_upstream_distance_limit
cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT = syscfg.workzone_downstream_distance_limit
cfg.WZ_UPSTREAM_DISTANCE_LIMIT = syscfg.workzone_upstream_distance_limit
cfg.SE_ARRIVAL_WINDOW = syscfg.specialevent_arrival_window
cfg.SE_DEPARTURE_WINDOW1 = syscfg.specialevent_departure_window1
cfg.SE_DEPARTURE_WINDOW2 = syscfg.specialevent_departure_window2
# faverolles 1/12/2020: Adding AdminClient MOE Config Parameters
cfg.MOE_CRITICAL_DENSITY = syscfg.moe_critical_density
cfg.MOE_LANE_CAPACITY = syscfg.moe_lane_capacity
cfg.MOE_CONGESTION_THRESHOLD_SPEED = syscfg.moe_congestion_threshold_speed
da.close_session()
def tetres_actionlog_list():
limit = request.form.get('limit', 100)
da = ActionLogDataAccess()
data_list = da.list(limit=limit, order_by=('id', 'desc'))
da.close_session()
da_config = ConfigDataAccess()
running_id_item = da_config.get_by_name(
cfg.OPT_NAME_ACTIONLOG_ID_IN_PROCESSING)
da_config.close_session()
if running_id_item and running_id_item.content:
running_id = int(running_id_item.content)
for item in data_list:
if item.id == running_id:
item.status = ActionLogDataAccess.STATUS_RUNNING
return prot.response_success({'list': data_list})
def set_system_config_info(syscfg):
"""
:rtype: pyticas_tetres.ttypes.SystemConfigInfo
"""
da = ConfigDataAccess()
inserted = da.insert_or_update(cfg.OPT_NAME_SYSCONFIG, json.dumps(syscfg))
if inserted == False or not da.commit():
da.close_session()
return False
prev_syscfg = SystemConfigInfo()
prev_syscfg.data_archive_start_year = cfg.DATA_ARCHIVE_START_YEAR
prev_syscfg.daily_job_offset_days = cfg.DAILY_JOB_OFFSET_DAYS
prev_syscfg.daily_job_start_time = cfg.DAILY_JOB_START_TIME
prev_syscfg.weekly_job_start_day = cfg.WEEKLY_JOB_START_WEEKDAY
prev_syscfg.weekly_job_start_time = cfg.WEEKLY_JOB_START_TIME
prev_syscfg.monthly_job_start_date = cfg.MONTHLY_JOB_START_DAY
prev_syscfg.monthly_job_start_time = cfg.MONTHLY_JOB_START_TIME
prev_syscfg.incident_downstream_distance_limit = cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT
prev_syscfg.incident_upstream_distance_limit = cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT
prev_syscfg.workzone_downstream_distance_limit = cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT
prev_syscfg.workzone_upstream_distance_limit = cfg.WZ_UPSTREAM_DISTANCE_LIMIT
prev_syscfg.specialevent_arrival_window = cfg.SE_ARRIVAL_WINDOW
prev_syscfg.specialevent_departure_window1 = cfg.SE_DEPARTURE_WINDOW1
prev_syscfg.specialevent_departure_window2 = cfg.SE_DEPARTURE_WINDOW2
# faverolles 1/12/2020: Adding AdminClient MOE Config Parameters
prev_syscfg.moe_critical_density = cfg.MOE_CRITICAL_DENSITY
prev_syscfg.moe_lane_capacity = cfg.MOE_LANE_CAPACITY
prev_syscfg.moe_congestion_threshold_speed = cfg.MOE_CONGESTION_THRESHOLD_SPEED
cfg.DATA_ARCHIVE_START_YEAR = syscfg.data_archive_start_year
cfg.DAILY_JOB_OFFSET_DAYS = syscfg.daily_job_offset_days
cfg.DAILY_JOB_START_TIME = syscfg.daily_job_start_time
cfg.WEEKLY_JOB_START_WEEKDAY = syscfg.weekly_job_start_day
cfg.WEEKLY_JOB_START_TIME = syscfg.weekly_job_start_time
cfg.MONTHLY_JOB_START_DAY = syscfg.monthly_job_start_date
cfg.MONTHLY_JOB_START_TIME = syscfg.monthly_job_start_time
cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT = syscfg.incident_downstream_distance_limit
cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT = syscfg.incident_upstream_distance_limit
cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT = syscfg.workzone_downstream_distance_limit
cfg.WZ_UPSTREAM_DISTANCE_LIMIT = syscfg.workzone_upstream_distance_limit
cfg.SE_ARRIVAL_WINDOW = syscfg.specialevent_arrival_window
cfg.SE_DEPARTURE_WINDOW1 = syscfg.specialevent_departure_window1
cfg.SE_DEPARTURE_WINDOW2 = syscfg.specialevent_departure_window2
# faverolles 1/12/2020: Adding AdminClient MOE Config Parameters
cfg.MOE_CRITICAL_DENSITY = syscfg.moe_critical_density
cfg.MOE_LANE_CAPACITY = syscfg.moe_lane_capacity
cfg.MOE_CONGESTION_THRESHOLD_SPEED = syscfg.moe_congestion_threshold_speed
return prev_syscfg
def _initialize_actionlog_status():
""" Delete running action log id from `config` table and delete
:return:
:rtype:
"""
da_config = ConfigDataAccess()
da_config.insert_or_update(cfg.OPT_NAME_ACTIONLOG_ID_IN_PROCESSING, '')
da_config.commit()
da_config.close_session()
now = datetime.datetime.now()
da_actionlog = ActionLogDataAccess()
running_items = da_actionlog.list(
status=ActionLogDataAccess.STATUS_RUNNING)
for item in running_items:
da_actionlog.update(
item.id, {
'status': ActionLogDataAccess.STATUS_STOPPED,
'status_updated_date': now
})
da_actionlog.commit()
da_actionlog.close_session()
def run():
# faverolles 1/16/2020 NOTE: Fired when admin client issues
# /tetres/adm/syscfg/update
"""
:rtype:
"""
logger = getLogger(__name__)
OPT_NAME = cfg.OPT_NAME_ACTIONLOG_ID_IN_PROCESSING
da_config = ConfigDataAccess()
running_config = da_config.insert_or_update(OPT_NAME, '')
if running_config is False or not da_config.commit():
logger.warning('cannot update "%s" option' % OPT_NAME)
return
running_config_id = running_config.id
da_actionlog = ActionLogDataAccess()
action_logs = da_actionlog.list(start_time=None, handled=False)
handlers = {
ActionLogDataAccess.DT_TTROUTE: _handler_ttroute,
ActionLogDataAccess.DT_INCIDENT: _handler_incident,
ActionLogDataAccess.DT_WORKZONE: _handler_workzone,
ActionLogDataAccess.DT_SPECIALEVENT: _handler_specialevent,
ActionLogDataAccess.DT_SNOWMGMT: _handler_snowmanagement,
ActionLogDataAccess.DT_SNOWROUTE: _handler_snowroute,
ActionLogDataAccess.DT_SNOWEVENT: _handler_snowevent,
ActionLogDataAccess.DT_SYSTEMCONFIG: _handler_systemconfig,
}
logger.debug('>>> Doing post-processing of admin actions')
handled = []
for action_log in action_logs:
logger.debug(' - processing for %s ' % action_log)
key = '%s - %s' % (action_log.target_table, action_log.target_id)
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_RUNNING,
'status_updated_date': datetime.datetime.now()})
da_config.update(running_config_id, {'content': action_log.id})
if not da_config.commit():
da_config.rollback()
logger.warning('cannot update "%s" option' % OPT_NAME)
# skip if the item is already handled (user can modify the same data several times)
if key in handled and action_log.action_type in [ActionLogDataAccess.INSERT, ActionLogDataAccess.UPDATE]:
logger.debug(' : skip : already processed')
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
# find insance of data access module
da = tablefinder.get_da_instance_by_tablename(action_log.target_table)
if not da:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'Database Access module is not found',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
logger.warning(' : skip : cannot find database access module (tablename=%s)' % action_log.target_table)
continue
# retrieve target item
item = da.get_data_by_id(action_log.target_id)
# if item is deleted...
if not item:
logger.debug(' : skip : item is not found')
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'reason': 'target data is not found',
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
# proceed by data type
handler = handlers.get(action_log.target_datatype)
if not handler:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'handler for the data not found',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
logger.debug(' : skip : handler is not found')
continue
try:
reason = ''
is_handled = handler(da, item, action_log)
if isinstance(is_handled, tuple):
is_handled, reason = is_handled[0], is_handled[1]
except Exception as ex:
tb.traceback(ex)
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'exception occured during processing data',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
if is_handled:
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
if key != ActionLogDataAccess.DELETE:
handled.append(key)
else:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': reason if reason else 'target data is not handled',
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
if not da_actionlog.commit():
da_actionlog.rollback()
da_actionlog.close_session()
logger.debug(' - fail to update %s ' % action_log)
return
logger.debug(' : end of processing for %s ' % action_log)
da_actionlog.close_session()
da_config.close_session()
logger.debug('<<< End of post-processing of admin actions')
def put_task_to_actionlog(prev_syscfg):
"""
:type prev_syscfg: pyticas_tetres.ttypes.SystemConfigInfo
"""
da_config = ConfigDataAccess()
syscfg = da_config.get_by_name(cfg.OPT_NAME_SYSCONFIG)
if not syscfg:
getLogger(__name__).warning('Cannot find the updated system configuration from `config` table')
da_config.close_session()
return
should_run_actionlog_handler = False
da_actionlog = ActionLogDataAccess()
is_data_archive_start_year_extended = False
is_data_archive_start_year_shrinked = False
if cfg.DATA_ARCHIVE_START_YEAR != prev_syscfg.data_archive_start_year:
if cfg.DATA_ARCHIVE_START_YEAR < prev_syscfg.data_archive_start_year:
is_data_archive_start_year_extended = True
elif cfg.DATA_ARCHIVE_START_YEAR > prev_syscfg.data_archive_start_year:
is_data_archive_start_year_shrinked = True
# cancled already posted action logs
if is_data_archive_start_year_shrinked or is_data_archive_start_year_extended:
ex_logs = da_actionlog.search(
searches=[('target_datatype', ActionLogDataAccess.DT_SYSTEMCONFIG), ('handled', False)],
op='and',
cond='match',
as_model=True)
for a_log in ex_logs:
a_log.handled = True
a_log.handled_date = datetime.datetime.now()
a_log.status = 'Cancled due to another action'
a_log.status_updated_date = datetime.datetime.now()
da_actionlog.commit()
if is_data_archive_start_year_extended:
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DATA_ARCHIVE_START_YEAR_EXTENDED: %d -> %d' % (
prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
handled=False)
elif is_data_archive_start_year_shrinked:
# add log for re-calculation
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DATA_ARCHIVE_START_YEAR_SHRINKED: %d -> %d' % (
prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
handled=False)
if (cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_downstream_distance_limit
or cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_upstream_distance_limit):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_INCIDENT,
handled=False)
if (cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_downstream_distance_limit
or cfg.WZ_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_upstream_distance_limit):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_WORKZONE,
handled=False)
if (cfg.SE_ARRIVAL_WINDOW != prev_syscfg.specialevent_arrival_window
or cfg.SE_DEPARTURE_WINDOW1 != prev_syscfg.specialevent_departure_window1
or cfg.SE_DEPARTURE_WINDOW2 != prev_syscfg.specialevent_departure_window2):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_SPECIALEVENT,
handled=False)
# restart scheduler
if (cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time
or cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days
or cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day
or cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time
or cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date
or cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time):
scheduler.restart()
if cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DAILY_JOB_START_TIME is updated : %s -> %s' % (
prev_syscfg.daily_job_start_time, cfg.DAILY_JOB_START_TIME),
handled=True)
if cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DAILY_JOB_OFFSET_DAYS is updated: %s -> %s' % (
prev_syscfg.daily_job_offset_days, cfg.DAILY_JOB_OFFSET_DAYS),
handled=True)
if cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'WEEKLY_JOB_START_WEEKDAY is updated: %s -> %s' % (
prev_syscfg.weekly_job_start_day, cfg.WEEKLY_JOB_START_WEEKDAY),
handled=True)
if cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'WEEKLY_JOB_START_TIME is updated: %s -> %s' % (
prev_syscfg.weekly_job_start_time, cfg.WEEKLY_JOB_START_TIME),
handled=True)
if cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'MONTHLY_JOB_START_DAY is updated: %s -> %s' % (
prev_syscfg.monthly_job_start_date, cfg.MONTHLY_JOB_START_DAY),
handled=True)
if cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'MONTHLY_JOB_START_TIME is updated: %s -> %s' % (
prev_syscfg.monthly_job_start_time, cfg.MONTHLY_JOB_START_TIME),
handled=True)
if not should_run_actionlog_handler:
unhandled = da_actionlog.list(target_datatypes=[ActionLogDataAccess.DT_SYSTEMCONFIG], handled=False)
if unhandled:
should_run_actionlog_handler = True
da_actionlog.close_session()
da_config.close_session()
# add actionlog handler to the task queue in the worker process
if should_run_actionlog_handler:
getLogger(__name__).debug('System configurations are updated and the handler process is posted')
worker.add_task(actionlog_proc.run)
else:
getLogger(__name__).debug('System configurations are updated and the handler process is NOT posted')