def run():
if '_01_tt' not in sys.modules:
from pyticas_tetres.sched.daily_tasks import _01_tt, _02_load_weather, _03_load_incident, _04_tagging
periods = []
# faverolles 1/16/2020 NOTE: always starts at datetime.today
today = datetime.datetime.today()
target_day = today - datetime.timedelta(days=cfg.DAILY_JOB_OFFSET_DAYS)
last_day_with_tt_data = _find_last_date(today.year)
if last_day_with_tt_data and last_day_with_tt_data <= target_day:
periods = period.create_periods(last_day_with_tt_data.date(),
target_day.date(), '00:00:00',
'23:59:00', cfg.TT_DATA_INTERVAL)
else:
prd = period.create_period(target_day.strftime("%Y-%m-%d 00:00:00"),
target_day.strftime("%Y-%m-%d 23:59:00"),
cfg.TT_DATA_INTERVAL)
periods.append(prd)
try:
non_completed_dates_and_routes = _check_logs()
periods = [_prd for _route_id, _prd in non_completed_dates_and_routes
] + periods
except Exception as ex:
getLogger(__name__).warning(
'error occured when checking daily-processing log : %s' %
tb.traceback(ex, f_print=False))
periods = list(set(periods))
if len(periods) > N_LIMIT_OF_DAYS_TO_PROCESS:
getLogger(__name__).warning(
'too many days to process. please use data loader program to process the long-time periods'
)
return
for prd in periods:
getLogger(__name__).info('>> running daily task for %s' %
prd.get_date_string())
try:
_01_tt.run(prd)
_02_load_weather.run(prd)
_03_load_incident.run(prd)
_04_tagging.run(prd)
except Exception as ex:
tb.traceback(ex)
getLogger(__name__).warning(
'Exception occured while performing daily task')
def _moe(moe_func, moe_name, **kwargs):
"""
:type moe_func: callable
:type moe_name: str
:return:
"""
try:
route_json = request.form.get('route', None)
periods = request.form.get('periods', None)
if not route_json or not periods:
return prot.response_error('Invalid Parameter')
r = json2route(route_json)
period_list = []
for prdinfo in json.loads(periods):
prd = period.create_period(
(prdinfo['start_year'], prdinfo['start_month'], prdinfo['start_date'], prdinfo['start_hour'],
prdinfo['start_min']),
(prdinfo['end_year'], prdinfo['end_month'], prdinfo['end_date'], prdinfo['end_hour'],
prdinfo['end_min']),
prdinfo['interval']
)
period_list.append(prd)
tmp_dir = Infra.get_infra().get_path('moe_tmp', create=True)
uid = str(uuid.uuid4())
est_file = os.path.join(tmp_dir, '%s.xlsx' % uid)
res = moe_func(r, period_list)
write = kwargs.get('write_function', writer.write)
write(est_file, r, res, **kwargs)
encoded = None
with open(est_file, 'rb') as f:
xlsx_content = f.read()
encoded = base64.b64encode(xlsx_content)
if not encoded:
return prot.response_error('ERROR : %s' % moe_name)
os.remove(est_file)
return prot.response_success(obj=encoded.decode('utf-8'))
except Exception as ex:
tb.traceback(ex)
return prot.response_error('ERROR : %s' % moe_name)
def _dry_days_for_nighttime(target_station, months, interval):
""" return dry day list
:type months: list[(int, int)]
:type interval: int
:rtype: (list[pyticas.ttypes.Period], str)
"""
normal_days = []
weather_source = ''
weather_sources = []
today = datetime.date.today()
if setting.LATE_NIGHT_START_TIME > setting.LATE_NIGHT_END_TIME:
delta = 24 - setting.LATE_NIGHT_START_TIME + setting.LATE_NIGHT_END_TIME
else:
delta = setting.LATE_NIGHT_END_TIME - setting.LATE_NIGHT_START_TIME
for idx, (year, month) in enumerate(months):
sd = datetime.date(year, month, 1)
dayrange = calendar.monthrange(year, month)
last_day = datetime.date(year, month, dayrange[1])
while sd < last_day and sd < today:
if prd_helper.is_holiday(sd) or not _is_weekday(sd):
sd = sd + datetime.timedelta(days=1)
continue
sdatetime = datetime.datetime(sd.year, sd.month, sd.day, setting.LATE_NIGHT_START_TIME, 0)
ndatetime = sdatetime + datetime.timedelta(hours=delta)
prd = prd_helper.create_period((sd.year, sd.month, sd.day, setting.LATE_NIGHT_START_TIME, 0),
(ndatetime.year, ndatetime.month, ndatetime.day, setting.LATE_NIGHT_END_TIME, 0), interval)
is_dry, weather_source = is_dry_day(target_station, prd)
if is_dry:
normal_days.append(prd)
weather_sources.append(weather_source)
sd = sd + datetime.timedelta(days=1)
return (normal_days, weather_source)
def _dry_days_for_daytime(target_station, months, start_time, end_time, interval):
""" return dry day list
:type months: list[(int, int)]
:type start_time: datetime.time
:type end_time: datetime.time
:type interval: int
:rtype: (list[pyticas.ttypes.Period], str)
"""
logger = getLogger(__name__)
normal_days = []
weather_source = ''
weather_sources = []
today = datetime.date.today()
for idx, (year, month) in enumerate(months):
day = datetime.date(year, month, 1)
dayrange = calendar.monthrange(year, month)
last_day = datetime.date(year, month, dayrange[1])
while day < last_day and day < today:
if prd_helper.is_holiday(day) or not _is_weekday(day):
day = day + datetime.timedelta(days=1)
continue
prd = prd_helper.create_period((day.year, day.month, day.day, start_time.hour, start_time.minute),
(day.year, day.month, day.day, end_time.hour, end_time.minute), interval)
#logger.debug(' - checking : %s' % prd.get_period_string())
is_dry, weather_source = is_dry_day(target_station, prd)
#logger.debug(' > is_dry=%s, weather_station=%s' % (is_dry, weather_source))
if is_dry:
normal_days.append(prd)
weather_sources.append(weather_source)
day = day + datetime.timedelta(days=1)
return (normal_days, weather_source)