def test_sleep_task():
generator = iter([SleepIteration(sleep=1)])
task = ExecutorIterationTask(generator)
with pytest.raises(SleepException):
try:
list(task)
except SleepException as sleep_task:
assert sleep_task.sleep == 1
raise
def test_exeption_iteration_in_pool2(pools):
pools.append_pools({"testpool": 0})
generator = iter([NextIterationInPools(pool_names=["testpool"]), "item2"])
task = ExecutorIterationTask(generator)
with pytest.raises(PoolOverflowingException):
list(task)
pools.update_pools({"testpool": 1})
assert list(task) == ["item2"]
def test_exeption_iteration_in_pool(pools):
pools.append_pools({"testpool": 0})
generator = iter([NextIterationInPools(pool_names=["testpool"]), "item"])
task = ExecutorIterationTask(generator)
with pytest.raises(PoolOverflowingException):
try:
list(task)
except PoolOverflowingException as sleep_task:
assert sleep_task.sleep == 1
raise
def task(self, *args, **kwargs) -> ExecutorIterationTask:
"""Flow wrapper for scheduler and executor."""
if self._task is not False:
raise PermissionError(
"For one class object, you can create one task")
self._task = True
return ExecutorIterationTask(
self._iterator(*args, **kwargs),
expires=self.Work.expires,
soft_time_limit_seconds=self.Work.soft_time_limit_seconds,
)
def order_task(*args, **kwargs):
count_flows = 4
worktimes = [
pendulum.datetime(2021, 1, i + 1) for i in range(count_flows)
]
for worktime in worktimes:
NOTEBOOK.load.file_name = (
f"{test_thread_executor_yandex_metrika_logs.__name__}.tsv")
flow = ETLOperator(NOTEBOOK)
generator = flow(start_period=worktime, end_period=worktime)
yield ExecutorIterationTask(generator)
def test_soft_time_limit():
generator = (time.sleep(2) for i in range(1))
task = ExecutorIterationTask(generator, soft_time_limit_seconds=1)
with pytest.raises(SoftTimeLimitError):
list(task)
def test_expires():
generator = (time.sleep(1) for i in range(1))
task = ExecutorIterationTask(generator, expires=pendulum.now())
with pytest.raises(ExpiredError):
list(task)
def ordering_task(*args, **kwargs):
for _ in range(count_task):
yield ExecutorIterationTask(flow())
def task(self, *args, **kwargs) -> ExecutorIterationTask:
"""Flow wrapper for scheduler and executor."""
return ExecutorIterationTask(self(*args, **kwargs),
expires=None,
soft_time_limit_seconds=None)