def add(action_type,
datatype,
tablename,
target_id,
data_desc,
handled=False,
dbsession=None,
commit=True):
"""
:type action_type: str
:type datatype: str
:type tablename: str
:type target_id: int
:type data_desc: str
:type handled: bool
:type dbsession: sqlalchemy.orm.Session
:type commit: bool
:rtype; Union(pyticas_tetres.db.model.ActionLog, False)
"""
if dbsession:
da_actionlog = ActionLogDataAccess(session=dbsession)
else:
da_actionlog = ActionLogDataAccess()
a = ActionLogInfo()
a.action_type = action_type
a.target_datatype = datatype
a.target_table = tablename
a.target_id = int(target_id) if target_id else None
a.data_desc = data_desc
a.handled = handled
a.user_ip = request.remote_addr
a.handled_date = None if not handled else datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
# print('Action Log ====================')
# import pprint
# pprint.pprint(a.get_dict())
# print('================================')
inserted = da_actionlog.insert(a, print_exception=True)
if commit or not dbsession:
da_actionlog.commit()
def _initialize_actionlog_status():
""" Delete running action log id from `config` table and delete
:return:
:rtype:
"""
da_config = ConfigDataAccess()
da_config.insert_or_update(cfg.OPT_NAME_ACTIONLOG_ID_IN_PROCESSING, '')
da_config.commit()
da_config.close_session()
now = datetime.datetime.now()
da_actionlog = ActionLogDataAccess()
running_items = da_actionlog.list(
status=ActionLogDataAccess.STATUS_RUNNING)
for item in running_items:
da_actionlog.update(
item.id, {
'status': ActionLogDataAccess.STATUS_STOPPED,
'status_updated_date': now
})
da_actionlog.commit()
da_actionlog.close_session()
def run():
# faverolles 1/16/2020 NOTE: Fired when admin client issues
# /tetres/adm/syscfg/update
"""
:rtype:
"""
logger = getLogger(__name__)
OPT_NAME = cfg.OPT_NAME_ACTIONLOG_ID_IN_PROCESSING
da_config = ConfigDataAccess()
running_config = da_config.insert_or_update(OPT_NAME, '')
if running_config is False or not da_config.commit():
logger.warning('cannot update "%s" option' % OPT_NAME)
return
running_config_id = running_config.id
da_actionlog = ActionLogDataAccess()
action_logs = da_actionlog.list(start_time=None, handled=False)
handlers = {
ActionLogDataAccess.DT_TTROUTE: _handler_ttroute,
ActionLogDataAccess.DT_INCIDENT: _handler_incident,
ActionLogDataAccess.DT_WORKZONE: _handler_workzone,
ActionLogDataAccess.DT_SPECIALEVENT: _handler_specialevent,
ActionLogDataAccess.DT_SNOWMGMT: _handler_snowmanagement,
ActionLogDataAccess.DT_SNOWROUTE: _handler_snowroute,
ActionLogDataAccess.DT_SNOWEVENT: _handler_snowevent,
ActionLogDataAccess.DT_SYSTEMCONFIG: _handler_systemconfig,
}
logger.debug('>>> Doing post-processing of admin actions')
handled = []
for action_log in action_logs:
logger.debug(' - processing for %s ' % action_log)
key = '%s - %s' % (action_log.target_table, action_log.target_id)
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_RUNNING,
'status_updated_date': datetime.datetime.now()})
da_config.update(running_config_id, {'content': action_log.id})
if not da_config.commit():
da_config.rollback()
logger.warning('cannot update "%s" option' % OPT_NAME)
# skip if the item is already handled (user can modify the same data several times)
if key in handled and action_log.action_type in [ActionLogDataAccess.INSERT, ActionLogDataAccess.UPDATE]:
logger.debug(' : skip : already processed')
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
# find insance of data access module
da = tablefinder.get_da_instance_by_tablename(action_log.target_table)
if not da:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'Database Access module is not found',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
logger.warning(' : skip : cannot find database access module (tablename=%s)' % action_log.target_table)
continue
# retrieve target item
item = da.get_data_by_id(action_log.target_id)
# if item is deleted...
if not item:
logger.debug(' : skip : item is not found')
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'reason': 'target data is not found',
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
# proceed by data type
handler = handlers.get(action_log.target_datatype)
if not handler:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'handler for the data not found',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
logger.debug(' : skip : handler is not found')
continue
try:
reason = ''
is_handled = handler(da, item, action_log)
if isinstance(is_handled, tuple):
is_handled, reason = is_handled[0], is_handled[1]
except Exception as ex:
tb.traceback(ex)
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': 'exception occured during processing data',
'status_updated_date': datetime.datetime.now()})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
continue
if is_handled:
da_actionlog.update(action_log.id, {'handled': True,
'handled_date': datetime.datetime.now(),
'status': ActionLogDataAccess.STATUS_DONE,
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
if key != ActionLogDataAccess.DELETE:
handled.append(key)
else:
da_actionlog.update(action_log.id, {'status': ActionLogDataAccess.STATUS_FAIL,
'reason': reason if reason else 'target data is not handled',
'status_updated_date': datetime.datetime.now()
})
da_actionlog.commit()
da_config.update(running_config_id, {'content': ''})
da_config.commit()
if not da_actionlog.commit():
da_actionlog.rollback()
da_actionlog.close_session()
logger.debug(' - fail to update %s ' % action_log)
return
logger.debug(' : end of processing for %s ' % action_log)
da_actionlog.close_session()
da_config.close_session()
logger.debug('<<< End of post-processing of admin actions')
def put_task_to_actionlog(prev_syscfg):
"""
:type prev_syscfg: pyticas_tetres.ttypes.SystemConfigInfo
"""
da_config = ConfigDataAccess()
syscfg = da_config.get_by_name(cfg.OPT_NAME_SYSCONFIG)
if not syscfg:
getLogger(__name__).warning('Cannot find the updated system configuration from `config` table')
da_config.close_session()
return
should_run_actionlog_handler = False
da_actionlog = ActionLogDataAccess()
is_data_archive_start_year_extended = False
is_data_archive_start_year_shrinked = False
if cfg.DATA_ARCHIVE_START_YEAR != prev_syscfg.data_archive_start_year:
if cfg.DATA_ARCHIVE_START_YEAR < prev_syscfg.data_archive_start_year:
is_data_archive_start_year_extended = True
elif cfg.DATA_ARCHIVE_START_YEAR > prev_syscfg.data_archive_start_year:
is_data_archive_start_year_shrinked = True
# cancled already posted action logs
if is_data_archive_start_year_shrinked or is_data_archive_start_year_extended:
ex_logs = da_actionlog.search(
searches=[('target_datatype', ActionLogDataAccess.DT_SYSTEMCONFIG), ('handled', False)],
op='and',
cond='match',
as_model=True)
for a_log in ex_logs:
a_log.handled = True
a_log.handled_date = datetime.datetime.now()
a_log.status = 'Cancled due to another action'
a_log.status_updated_date = datetime.datetime.now()
da_actionlog.commit()
if is_data_archive_start_year_extended:
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DATA_ARCHIVE_START_YEAR_EXTENDED: %d -> %d' % (
prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
handled=False)
elif is_data_archive_start_year_shrinked:
# add log for re-calculation
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DATA_ARCHIVE_START_YEAR_SHRINKED: %d -> %d' % (
prev_syscfg.data_archive_start_year, cfg.DATA_ARCHIVE_START_YEAR),
handled=False)
if (cfg.INCIDENT_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_downstream_distance_limit
or cfg.INCIDENT_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.incident_upstream_distance_limit):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_INCIDENT,
handled=False)
if (cfg.WZ_DOWNSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_downstream_distance_limit
or cfg.WZ_UPSTREAM_DISTANCE_LIMIT != prev_syscfg.workzone_upstream_distance_limit):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_WORKZONE,
handled=False)
if (cfg.SE_ARRIVAL_WINDOW != prev_syscfg.specialevent_arrival_window
or cfg.SE_DEPARTURE_WINDOW1 != prev_syscfg.specialevent_departure_window1
or cfg.SE_DEPARTURE_WINDOW2 != prev_syscfg.specialevent_departure_window2):
should_run_actionlog_handler = True
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
ActionLogDataAccess.DT_SPECIALEVENT,
handled=False)
# restart scheduler
if (cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time
or cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days
or cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day
or cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time
or cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date
or cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time):
scheduler.restart()
if cfg.DAILY_JOB_START_TIME != prev_syscfg.daily_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DAILY_JOB_START_TIME is updated : %s -> %s' % (
prev_syscfg.daily_job_start_time, cfg.DAILY_JOB_START_TIME),
handled=True)
if cfg.DAILY_JOB_OFFSET_DAYS != prev_syscfg.daily_job_offset_days:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'DAILY_JOB_OFFSET_DAYS is updated: %s -> %s' % (
prev_syscfg.daily_job_offset_days, cfg.DAILY_JOB_OFFSET_DAYS),
handled=True)
if cfg.WEEKLY_JOB_START_WEEKDAY != prev_syscfg.weekly_job_start_day:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'WEEKLY_JOB_START_WEEKDAY is updated: %s -> %s' % (
prev_syscfg.weekly_job_start_day, cfg.WEEKLY_JOB_START_WEEKDAY),
handled=True)
if cfg.WEEKLY_JOB_START_TIME != prev_syscfg.weekly_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'WEEKLY_JOB_START_TIME is updated: %s -> %s' % (
prev_syscfg.weekly_job_start_time, cfg.WEEKLY_JOB_START_TIME),
handled=True)
if cfg.MONTHLY_JOB_START_DAY != prev_syscfg.monthly_job_start_date:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'MONTHLY_JOB_START_DAY is updated: %s -> %s' % (
prev_syscfg.monthly_job_start_date, cfg.MONTHLY_JOB_START_DAY),
handled=True)
if cfg.MONTHLY_JOB_START_TIME != prev_syscfg.monthly_job_start_time:
actionlog.add(ActionLogDataAccess.UPDATE,
ActionLogDataAccess.DT_SYSTEMCONFIG,
Config.__tablename__,
syscfg.id,
'MONTHLY_JOB_START_TIME is updated: %s -> %s' % (
prev_syscfg.monthly_job_start_time, cfg.MONTHLY_JOB_START_TIME),
handled=True)
if not should_run_actionlog_handler:
unhandled = da_actionlog.list(target_datatypes=[ActionLogDataAccess.DT_SYSTEMCONFIG], handled=False)
if unhandled:
should_run_actionlog_handler = True
da_actionlog.close_session()
da_config.close_session()
# add actionlog handler to the task queue in the worker process
if should_run_actionlog_handler:
getLogger(__name__).debug('System configurations are updated and the handler process is posted')
worker.add_task(actionlog_proc.run)
else:
getLogger(__name__).debug('System configurations are updated and the handler process is NOT posted')
try:
if route_ids:
initial_data_maker._calculate_tt_and_categorize(
sdate,
edate,
db_info=dbinfo.tetres_db_info(),
route_ids=route_ids)
with open(filename, 'a+') as f:
f.write('ended at ' +
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') +
'\n')
for an_actionlog in action_logs:
now = datetime.datetime.now()
da_actionlog.update(
an_actionlog.id, {
'handled': True,
'handled_date': now,
'status': ActionLogDataAccess.STATUS_DONE,
'status_updated_date': now,
'processed_start_date': sdate,
'processed_end_date': edate
})
da_actionlog.commit()
except Exception as ex:
print('exception:', ex)
with open(filename, 'a+') as f:
f.write('exception occured at ' +
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') +
'\n')