def validate_reexecution_memoization(plan_context: IPlanContext, execution_plan: ExecutionPlan) -> None: parent_run_id = plan_context.pipeline_run.parent_run_id check.opt_str_param(parent_run_id, "parent_run_id") if parent_run_id is None: return if not plan_context.instance.has_run(parent_run_id): raise DagsterRunNotFoundError( "Run id {} set as parent run id was not found in instance".format( parent_run_id), invalid_run_id=parent_run_id, ) # exclude full pipeline re-execution if len(execution_plan.step_keys_to_execute) == len(execution_plan.steps): return if execution_plan.artifacts_persisted: return raise DagsterInvariantViolationError( "Cannot perform reexecution with in-memory io managers.\n" "To enable reexecution, you can set a persistent io manager, such as the " 'fs_io_manager, in the resource_defs argument on your ModeDefinition: resource_defs={"io_manager": fs_io_manager}' )
def _check_reexecution_config(pipeline_context, execution_plan, run_config): check.invariant(pipeline_context.run_storage) if not pipeline_context.run_storage.is_persistent: raise DagsterInvariantViolationError( 'Cannot perform reexecution with non persistent run storage.') previous_run_id = run_config.reexecution_config.previous_run_id if not pipeline_context.run_storage.has_run(previous_run_id): raise DagsterRunNotFoundError( 'Run id {} set as previous run id was not found in run storage'. format(previous_run_id), invalid_run_id=previous_run_id, ) for step_output_handle in run_config.reexecution_config.step_output_handles: if not execution_plan.has_step(step_output_handle.step_key): raise DagsterExecutionStepNotFoundError( ('Step {step_key} was specified as a step from a previous run. ' 'It does not exist.').format( step_key=step_output_handle.step_key), step_key=step_output_handle.step_key, ) step = execution_plan.get_step_by_key(step_output_handle.step_key) if not step.has_step_output(step_output_handle.output_name): raise DagsterStepOutputNotFoundError( ('You specified a step_output_handle in the ReexecutionConfig that does ' 'not exist: Step {step_key} does not have output {output_name}.' ).format(step_key=step_output_handle.step_key, output_name=step_output_handle.output_name), step_key=step_output_handle.step_key, output_name=step_output_handle.output_name, )
def validate_reexecution_memoization(plan_context, execution_plan): check.inst_param(plan_context, "plan_context", IPlanContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) parent_run_id = plan_context.pipeline_run.parent_run_id check.opt_str_param(parent_run_id, "parent_run_id") if parent_run_id is None: return if not plan_context.instance.has_run(parent_run_id): raise DagsterRunNotFoundError( "Run id {} set as parent run id was not found in instance".format( parent_run_id), invalid_run_id=parent_run_id, ) # exclude full pipeline re-execution if len(execution_plan.step_keys_to_execute) == len(execution_plan.steps): return if execution_plan.artifacts_persisted: return raise DagsterInvariantViolationError( "Cannot perform reexecution with in-memory io managers.\n" "You may have configured non persistent intermediate storage `{}` for reexecution. " "Intermediate Storage is deprecated in 0.10.0 and will be removed in a future release." .format(plan_context.intermediate_storage.__class__.__name__))
def validate_reexecution_memoization(pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) parent_run_id = pipeline_context.pipeline_run.parent_run_id check.opt_str_param(parent_run_id, 'parent_run_id') if parent_run_id is None: return if not pipeline_context.instance.has_run(parent_run_id): raise DagsterRunNotFoundError( 'Run id {} set as parent run id was not found in instance'.format( parent_run_id), invalid_run_id=parent_run_id, ) # exclude full pipeline re-execution if len(execution_plan.step_keys_to_execute) == len(execution_plan.steps): return if not pipeline_context.intermediates_manager.is_persistent: raise DagsterInvariantViolationError( 'Cannot perform reexecution with non persistent intermediates manager `{}`.' .format(pipeline_context.intermediates_manager.__class__.__name__))
def validate_reexecution_memoization(pipeline_context, execution_plan): check.inst_param(pipeline_context, "pipeline_context", SystemExecutionContext) check.inst_param(execution_plan, "execution_plan", ExecutionPlan) parent_run_id = pipeline_context.pipeline_run.parent_run_id check.opt_str_param(parent_run_id, "parent_run_id") if parent_run_id is None: return if not pipeline_context.instance.has_run(parent_run_id): raise DagsterRunNotFoundError( "Run id {} set as parent run id was not found in instance".format( parent_run_id), invalid_run_id=parent_run_id, ) # exclude full pipeline re-execution if len(execution_plan.step_keys_to_execute) == len(execution_plan.steps): return # remove this once intermediate storage is fully deprecated # https://github.com/dagster-io/dagster/issues/3043 if pipeline_context.intermediate_storage.is_persistent: return # exclude the case where non-in-memory asset stores are configured on the required steps if check_all_asset_stores_non_mem_for_reexecution(pipeline_context, execution_plan) is False: raise DagsterInvariantViolationError( "Cannot perform reexecution with in-memory asset stores.\n" "You may have configured non persistent intermediate storage `{}` for reexecution. " "Intermediate Storage is deprecated in 0.10.0 and will be removed in 0.11.0." .format(pipeline_context.intermediate_storage.__class__.__name__))
def validate_retry_memoization(pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) parent_run_id = pipeline_context.pipeline_run.parent_run_id check.opt_str_param(parent_run_id, 'parent_run_id') if parent_run_id is None: return if not pipeline_context.instance.has_run(parent_run_id): raise DagsterRunNotFoundError( 'Run id {} set as parent run id was not found in instance'.format( parent_run_id), invalid_run_id=parent_run_id, ) if len(execution_plan.step_keys_to_execute) == len(execution_plan.steps): # this is a short-term proxy to distinguish between re-execution and retries. # Resume/retry will always have a subset of the execution plan, and re-execution will # always be the full execution plan (until we change dagit to enable re-execution subsets) return if not pipeline_context.intermediates_manager.is_persistent: raise DagsterInvariantViolationError( 'Cannot perform reexecution with non persistent intermediates manager `{}`.' .format(pipeline_context.intermediates_manager.__class__.__name__))
def get_run_group(self, run_id: str) -> Optional[Tuple[str, Iterable[PipelineRun]]]: check.str_param(run_id, "run_id") pipeline_run = self.get_run_by_id(run_id) if not pipeline_run: raise DagsterRunNotFoundError( f"Run {run_id} was not found in instance.", invalid_run_id=run_id ) # find root_run root_run_id = pipeline_run.root_run_id if pipeline_run.root_run_id else pipeline_run.run_id root_run = self.get_run_by_id(root_run_id) if not root_run: raise DagsterRunNotFoundError( f"Run id {root_run} set as root run id for run {run_id} was not found in instance.", invalid_run_id=root_run, ) # root_run_id to run_id 1:1 mapping # https://github.com/dagster-io/dagster/issues/2495 # Note: we currently use tags to persist the run group info root_to_run = ( db.select( [RunTagsTable.c.value.label("root_run_id"), RunTagsTable.c.run_id.label("run_id")] ) .where( db.and_(RunTagsTable.c.key == ROOT_RUN_ID_TAG, RunTagsTable.c.value == root_run_id) ) .alias("root_to_run") ) # get run group run_group_query = ( db.select([RunsTable.c.run_body]) .select_from( root_to_run.join( RunsTable, root_to_run.c.run_id == RunsTable.c.run_id, isouter=True, ) ) .alias("run_group") ) with self.connect() as conn: res = conn.execute(run_group_query) run_group = self._rows_to_runs(res) return (root_run_id, [root_run] + run_group)
def add_run_tags(self, run_id: str, new_tags: Dict[str, str]): check.str_param(run_id, "run_id") check.dict_param(new_tags, "new_tags", key_type=str, value_type=str) run = self.get_run_by_id(run_id) if not run: raise DagsterRunNotFoundError( f"Run {run_id} was not found in instance.", invalid_run_id=run_id) current_tags = run.tags if run.tags else {} all_tags = merge_dicts(current_tags, new_tags) partition = all_tags.get(PARTITION_NAME_TAG) partition_set = all_tags.get(PARTITION_SET_TAG) with self.connect() as conn: conn.execute(RunsTable.update() # pylint: disable=no-value-for-parameter .where(RunsTable.c.run_id == run_id).values( run_body=serialize_dagster_namedtuple( run.with_tags( merge_dicts(current_tags, new_tags))), partition=partition, partition_set=partition_set, update_timestamp=pendulum.now("UTC"), )) current_tags_set = set(current_tags.keys()) new_tags_set = set(new_tags.keys()) existing_tags = current_tags_set & new_tags_set added_tags = new_tags_set.difference(existing_tags) for tag in existing_tags: conn.execute(RunTagsTable.update() # pylint: disable=no-value-for-parameter .where( db.and_(RunTagsTable.c.run_id == run_id, RunTagsTable.c.key == tag)).values( value=new_tags[tag])) if added_tags: conn.execute( RunTagsTable.insert(), # pylint: disable=no-value-for-parameter [ dict(run_id=run_id, key=tag, value=new_tags[tag]) for tag in added_tags ], )
def get_run_group(self, run_id: str) -> Optional[Tuple[str, List[PipelineRun]]]: check.str_param(run_id, "run_id") pipeline_run = self._runs.get(run_id) if not pipeline_run: raise DagsterRunNotFoundError( f"Run {run_id} was not found in instance.", invalid_run_id=run_id) # if the run doesn't have root_run_id, itself is the root root_run = (self.get_run_by_id(pipeline_run.root_run_id) if pipeline_run.root_run_id else pipeline_run) if not root_run: return None run_group = [root_run] for curr_run in self._runs.values(): if curr_run.root_run_id == root_run.run_id: run_group.append(curr_run) return (root_run.root_run_id, run_group)
def validate_retry_memoization(pipeline_context, execution_plan): check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext) check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) if not execution_plan.previous_run_id: return if not pipeline_context.intermediates_manager.is_persistent: raise DagsterInvariantViolationError( 'Cannot perform reexecution with non persistent intermediates manager `{}`.'.format( pipeline_context.intermediates_manager.__class__.__name__ ) ) previous_run_id = execution_plan.previous_run_id if not pipeline_context.instance.has_run(previous_run_id): raise DagsterRunNotFoundError( 'Run id {} set as previous run id was not found in instance'.format(previous_run_id), invalid_run_id=previous_run_id, )
def _run_in_subprocess( serialized_execute_run_args, recon_pipeline, termination_event, subprocess_status_handler, run_event_handler, ): start_termination_thread(termination_event) try: execute_run_args = deserialize_json_to_dagster_namedtuple( serialized_execute_run_args) check.inst_param(execute_run_args, "execute_run_args", ExecuteExternalPipelineArgs) instance = DagsterInstance.from_ref(execute_run_args.instance_ref) pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id) if not pipeline_run: raise DagsterRunNotFoundError( "gRPC server could not load run {run_id} in order to execute it. Make sure that the gRPC server has access to your run storage." .format(run_id=execute_run_args.pipeline_run_id), invalid_run_id=execute_run_args.pipeline_run_id, ) pid = os.getpid() except: # pylint: disable=bare-except serializable_error_info = serializable_error_info_from_exc_info( sys.exc_info()) event = IPCErrorMessage( serializable_error_info=serializable_error_info, message="Error during RPC setup for executing run: {message}". format(message=serializable_error_info.message), ) subprocess_status_handler(event) subprocess_status_handler(RunInSubprocessComplete()) if instance: instance.dispose() return subprocess_status_handler(StartRunInSubprocessSuccessful()) run_event_handler( instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), )) # This is so nasty but seemingly unavoidable # https://amir.rachum.com/blog/2017/03/03/generator-cleanup/ closed = False try: for event in _core_execute_run(recon_pipeline, pipeline_run, instance): run_event_handler(event) except GeneratorExit: closed = True raise finally: if not closed: run_event_handler( instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format( pid=pid), pipeline_run, )) subprocess_status_handler(RunInSubprocessComplete()) instance.dispose()