def test_allow_execute_flow():
FlowItem.delete().where(FlowItem.name == FLOW_NAME).execute()
worktime = pendulum.datetime(2020, 1, 6, tz="Europe/Moscow")
interval_timedelta = dt.timedelta(1)
worktime_list = iter_range_datetime(
start_time=worktime - dt.timedelta(3),
end_time=worktime,
timedelta=interval_timedelta,
)
FlowItem.create_items(
flow_name=FLOW_NAME,
worktime_list=worktime_list,
status=Statuses.fatal_error,
notebook_hash="",
)
assert (FlowItem.allow_execute_flow(
FLOW_NAME, notebook_hash="", max_fatal_errors=3) is False)
assert (FlowItem.allow_execute_flow(
FLOW_NAME, notebook_hash="new", max_fatal_errors=3) is True)
FlowItem.recreate_prev_items(
flow_name=FLOW_NAME,
worktime=worktime,
offset_periods=10,
interval_timedelta=interval_timedelta,
)
assert (FlowItem.allow_execute_flow(
FLOW_NAME, notebook_hash="", max_fatal_errors=3) is True)
def create_missing_items(
cls,
flow_name: str,
start_time: dt.datetime,
end_time: dt.datetime,
interval_timedelta: dt.timedelta,
) -> list["FlowItem"]:
items = []
for datetime_ in iter_range_datetime(start_time, end_time,
interval_timedelta):
try:
item = cls.create(**{
cls.name.name: flow_name,
cls.worktime.name: datetime_
})
except peewee.IntegrityError:
item = cls.get(cls.name == flow_name,
cls.worktime == datetime_)
else:
logger.info("Created missing worktime {} for {}", datetime_,
flow_name)
items.append(item)
return items
def __call__(
self,
start_period: dt.datetime,
end_period: dt.datetime,
*,
async_mode: bool = False,
dry_run: bool = False,
**kwargs,
) -> Iterator[Union[dict, Optional[AsyncTaskT]]]:
date_log = self._get_period_text(start_period, end_period)
self.logger.update(self.name, filename=f"{date_log}.log", level=self.loglevel)
self.logger.info("Start flow: %s %s", self.name, date_log)
datetime_list = iter_range_datetime(
start_period, end_period, self.Work.interval_timedelta
)
self.items = self.Model.create_items(
self.name,
datetime_list,
operator=FlowOperator.etl,
data=self.operator_context.dict(exclude_unset=True),
)
log_data = {}
try:
for item in self.iterator(start_period, end_period, **kwargs):
if isinstance(item, (SleepTask, TaskPool)):
if async_mode is False:
if isinstance(item, SleepTask):
time.sleep(item.sleep)
continue
else:
log_data = item
self.Model.update_items(self.items, **log_data)
self.logger.debug("%s: %s", self.name, log_data)
yield item
except:
self.logger.exception("Fail flow: %s %s", self.name, date_log)
self.send_notifications(
**{
"status": FlowStatus.error,
"period": self._get_period_text(start_period, end_period),
**log_data,
}
)
raise
else:
self.send_notifications(
FlowStatus.success,
period=self._get_period_text(start_period, end_period),
)
finally:
self.Model.update_items(self.items)
self.logger.info("End flow: %s %s", self.name, date_log)
def current_worktime(self) -> dt.datetime:
"""
Returns the work datetime at the moment.
"""
end_time = pendulum.now(self.timezone)
if self.start_datetime > end_time:
start_time = self.start_datetime - self.interval_timedelta
if start_time > end_time:
raise ValueError(f"{start_time=} > {end_time=}")
else:
start_time = self.start_datetime
worktime = list(
iter_range_datetime(
start_time=start_time,
end_time=end_time,
timedelta=self.interval_timedelta,
))[-1]
if self.is_second_interval is False:
worktime = worktime - self.interval_timedelta
return worktime
def current_worktime(self) -> dt.datetime:
"""
Returns the work datetime at the moment.
"""
end_time = pendulum.now(self.timezone).at(
hour=self.start_datetime.hour,
minute=self.start_datetime.minute,
second=self.start_datetime.second,
)
if self.is_second_interval is False:
end_time = end_time - self.interval_timedelta
if self.start_datetime > end_time:
start_time = self.start_datetime - self.interval_timedelta
else:
start_time = self.start_datetime
return list(
iter_range_datetime(
start_time=start_time,
end_time=end_time,
timedelta=self.interval_timedelta,
)
)[-1]
def _iterator(
self,
start_period: dt.datetime,
end_period: dt.datetime,
*,
dry_run: bool = False,
**kwargs,
) -> Iterator[Union[dict, Optional[AsyncIterationT]]]:
self.operator_context.start_period = start_period
self.operator_context.end_period = end_period
self.Load.update_context(self.operator_context)
self.add_logger_file(dry_run)
period_text = self._get_period_text(start_period, end_period)
datetime_list = iter_range_datetime(start_period, end_period,
self.Work.interval_timedelta)
# TODO: Записи уже должны быть, их не надо создавать
self.items = self.Model.create_items(
self.notebook.name,
datetime_list,
**{
self.Model.data.name:
self.operator_context.dict(exclude_unset=True),
self.Model.logpath.name:
str(self.get_logfile_path().absolute()),
},
)
log_data = {}
try:
self.logger.info("Start flow: {} {}", self.notebook.name,
period_text)
for item in self(start_period,
end_period,
dry_run=dry_run,
**kwargs):
if isinstance(item, dict):
log_data = item
self.Model.update_items(self.items, **log_data)
self.logger.info("{}: {}", self.notebook.name, log_data)
if isinstance(item, (SleepIteration, NextIterationInPools)):
yield item
except Exception:
self.logger.exception("Fail flow: {} {}", self.notebook.name,
period_text)
if dry_run is False:
self.send_notifications(
**{
"status": Statuses.error,
"period": self._get_period_text(
start_period, end_period),
**log_data,
})
raise
else:
if dry_run is False:
self.send_notifications(
Statuses.success,
period=self._get_period_text(start_period, end_period),
)
finally:
self.Model.update_items(self.items)
self.logger.info("End flow: {} {}", self.notebook.name,
period_text)
