def restart( name: str, start_time: dt.datetime = typer.Option(..., "--start_time", "-s"), end_time: dt.datetime = typer.Option(..., "--end_time", "-e"), ): for name_ in name.split(","): if start_time or end_time: # Apply timezone. for file_name, config in YamlHelper.iter_parse_file_from_dir( FLOW_CONFIGS_DIR, match=name_): tz = config["work"]["schedule"]["timezone"] if start_time: start_time = start_time.replace( tzinfo=pendulum.timezone(tz)) if end_time: end_time = end_time.replace(tzinfo=pendulum.timezone(tz)) break count = FlowItem.change_status(name_, new_status=FlowStatus.add, from_time=start_time, to_time=end_time) typer.secho( f" {name_} {typer.style(f'{count=}', fg=typer.colors.WHITE)} OK")
def restart_errors(name: str): for name_ in name.split(","): count = FlowItem.change_status( name_, new_status=FlowStatus.add, filter_statuses=FlowStatus.error_statuses) typer.secho( f" {name_} {typer.style(f'{count=}', fg=typer.colors.WHITE)} OK")
def prepare_items_for_order(flow: "BaseOperator", start_period: dt.datetime, end_period: dt.datetime): # The status is changed so that there is no repeated ordering of tasks. FlowItem.change_status( flow.notebook.name, new_status=Statuses.run, from_time=start_period, to_time=end_period, ) if flow.Work.expires is not None: FlowItem.change_expires( flow.notebook.name, expires=flow.Work.expires, from_time=start_period, to_time=end_period, ) yield
def order_etl_flow( *, logger: Logger, async_mode: bool = False, dry_run: bool = False ) -> Iterator: """Prepare flow function to be sent to the queue and executed""" from flowmaster.operators.etl.service import ETLOperator from flowmaster.operators.etl.policy import ETLFlowConfig for file_name, config in YamlHelper.iter_parse_file_from_dir( FLOW_CONFIGS_DIR, match=".etl.flow" ): if dry_run: if config.get("provider") != "fakedata": continue try: flow_config = ETLFlowConfig(name=file_name, **config) except pydantic.ValidationError as exc: logger.error("ValidationError: '%s': %s", file_name, exc) continue except Exception as exc: logger.error("Error: '%s': %s", file_name, exc) continue work = ETLWork(flow_config) for start_period, end_period in work.iter_period_for_execute(): etl_flow = ETLOperator(flow_config) etl_flow_iterator = etl_flow( start_period, end_period, async_mode=async_mode, dry_run=dry_run ) # The status is changed so that there is no repeated ordering of tasks. FlowItem.change_status( etl_flow.name, new_status=FlowStatus.run, from_time=start_period, to_time=end_period, ) logger.info( "Order ETL flow [%s]: %s %s", etl_flow.name, start_period, end_period ) yield etl_flow_iterator
def test_change_status(): FlowItem.delete().where(FlowItem.name == FLOW_NAME).execute() worktime = pendulum.datetime(2020, 1, 6, tz="Europe/Moscow") interval_timedelta = dt.timedelta(1) items = FlowItem.create_missing_items( flow_name=FLOW_NAME, start_time=worktime - dt.timedelta(5), end_time=worktime, interval_timedelta=interval_timedelta, ) FlowItem.change_status( FLOW_NAME, new_status=FlowStatus.success, from_time=worktime - dt.timedelta(5), to_time=worktime, ) assert FlowItem.count_items(FLOW_NAME, statuses=[FlowStatus.success]) == len(items)