def get_external_schedule_execution( recon_repo, instance_ref, schedule_name, scheduled_execution_timestamp, scheduled_execution_timezone, ): check.inst_param( recon_repo, "recon_repo", ReconstructableRepository, ) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def(schedule_name) scheduled_execution_time = (pendulum.from_timestamp( scheduled_execution_timestamp, tz=scheduled_execution_timezone, ) if scheduled_execution_timestamp else None) schedule_context = ScheduleExecutionContext(instance_ref, scheduled_execution_time) try: with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution function for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): return ExternalScheduleExecutionData.from_execution_data( schedule_def.get_execution_data(schedule_context)) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def get_external_schedule_execution(external_schedule_execution_args): check.inst_param( external_schedule_execution_args, 'external_schedule_execution_args', ExternalScheduleExecutionArgs, ) recon_repo = recon_repository_from_origin( external_schedule_execution_args.repository_origin) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def( external_schedule_execution_args.schedule_name) instance = DagsterInstance.from_ref( external_schedule_execution_args.instance_ref) schedule_context = ScheduleExecutionContext(instance) try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of run_config_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) return ExternalScheduleExecutionData(run_config=run_config) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def get_external_schedule_execution(recon_repo, external_schedule_execution_args): check.inst_param( recon_repo, "recon_repo", ReconstructableRepository, ) check.inst_param( external_schedule_execution_args, "external_schedule_execution_args", ExternalScheduleExecutionArgs, ) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def( external_schedule_execution_args.schedule_name) with DagsterInstance.from_ref( external_schedule_execution_args.instance_ref) as instance: schedule_context = ScheduleExecutionContext(instance) schedule_execution_data_mode = external_schedule_execution_args.schedule_execution_data_mode try: with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of should_execute for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): should_execute = None if (schedule_execution_data_mode == ScheduleExecutionDataMode.LAUNCH_SCHEDULED_EXECUTION): should_execute = schedule_def.should_execute( schedule_context) if not should_execute: return ExternalScheduleExecutionData( should_execute=False, run_config=None, tags=None) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of run_config_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of tags_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): tags = schedule_def.get_tags(schedule_context) return ExternalScheduleExecutionData(run_config=run_config, tags=tags, should_execute=should_execute) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def get_external_schedule_execution( recon_repo, instance_ref, schedule_name, scheduled_execution_timestamp, scheduled_execution_timezone, ): check.inst_param( recon_repo, "recon_repo", ReconstructableRepository, ) definition = recon_repo.get_definition() schedule_def = definition.get_schedule_def(schedule_name) with DagsterInstance.from_ref(instance_ref) as instance: scheduled_execution_time = (pendulum.from_timestamp( scheduled_execution_timestamp, tz=scheduled_execution_timezone, ) if scheduled_execution_timestamp else None) schedule_context = ScheduleExecutionContext(instance, scheduled_execution_time) try: with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of should_execute for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): if not schedule_def.should_execute(schedule_context): return ExternalScheduleExecutionData(should_execute=False, run_config=None, tags=None) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of run_config_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) with user_code_error_boundary( ScheduleExecutionError, lambda: "Error occurred during the execution of tags_fn for schedule " "{schedule_name}".format(schedule_name=schedule_def.name), ): tags = schedule_def.get_tags(schedule_context) return ExternalScheduleExecutionData(run_config=run_config, tags=tags, should_execute=True) except ScheduleExecutionError: return ExternalScheduleExecutionErrorData( serializable_error_info_from_exc_info(sys.exc_info()))
def __init__(self, graphene_info, schedule_def): self._schedule_def = check.inst_param(schedule_def, 'schedule_def', ScheduleDefinition) self._schedule_context = ScheduleExecutionContext(graphene_info.context.instance) execution_params = schedule_def.execution_params environment_config = schedule_def.get_environment_dict(self._schedule_context) execution_params['environmentConfigData'] = environment_config super(DauphinScheduleDefinition, self).__init__( name=schedule_def.name, cron_schedule=schedule_def.cron_schedule, execution_params_string=seven.json.dumps(execution_params), )
def resolve_run_config_yaml(self, graphene_info): schedule_def = self._schedule_def schedule_context = ScheduleExecutionContext(graphene_info.context.instance) try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of environment_dict_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): environment_config = schedule_def.get_environment_dict(schedule_context) except ScheduleExecutionError: return None run_config_yaml = yaml.dump(environment_config, default_flow_style=False) return run_config_yaml if run_config_yaml else ''
def __init__(self, graphene_info, schedule_def): self._schedule_def = check.inst_param(schedule_def, 'schedule_def', ScheduleDefinition) self._schedule_context = ScheduleExecutionContext( graphene_info.context.instance) self._schedule_def = check.inst_param(schedule_def, 'schedule_def', ScheduleDefinition) super(DauphinScheduleDefinition, self).__init__( name=schedule_def.name, cron_schedule=schedule_def.cron_schedule, pipeline_name=schedule_def.selector.name, solid_subset=schedule_def.selector.solid_subset, mode=schedule_def.mode, )
def _launch_scheduled_execution(instance, schedule_def, pipeline, tick, stream): pipeline_def = pipeline.get_definition() # Run should_execute and halt if it returns False schedule_context = ScheduleExecutionContext(instance) with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of should_execute for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): should_execute = schedule_def.should_execute(schedule_context) if not should_execute: # Update tick to skipped state and return tick.update_with_status(ScheduleTickStatus.SKIPPED) stream.send(ScheduledExecutionSkipped()) return errors = [] run_config = {} schedule_tags = {} try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of run_config_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): run_config = schedule_def.get_run_config(schedule_context) except DagsterUserCodeExecutionError: error_data = serializable_error_info_from_exc_info(sys.exc_info()) errors.append(error_data) try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of tags_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): schedule_tags = schedule_def.get_tags(schedule_context) except DagsterUserCodeExecutionError: error_data = serializable_error_info_from_exc_info(sys.exc_info()) errors.append(error_data) pipeline_tags = pipeline_def.tags or {} check_tags(pipeline_tags, 'pipeline_tags') tags = merge_dicts(pipeline_tags, schedule_tags) mode = schedule_def.mode execution_plan_snapshot = None try: execution_plan = create_execution_plan( pipeline_def, run_config=run_config, mode=mode, ) execution_plan_snapshot = snapshot_from_execution_plan( execution_plan, pipeline_def.get_pipeline_snapshot_id()) except DagsterInvalidConfigError: error_data = serializable_error_info_from_exc_info(sys.exc_info()) errors.append(error_data) # Enter the run in the DB with the information we have possibly_invalid_pipeline_run = instance.create_run( pipeline_name=schedule_def.pipeline_name, run_id=None, run_config=run_config, mode=mode, solids_to_execute=pipeline.solids_to_execute, step_keys_to_execute=None, solid_selection=pipeline.solid_selection, status=None, root_run_id=None, parent_run_id=None, tags=tags, pipeline_snapshot=pipeline_def.get_pipeline_snapshot(), execution_plan_snapshot=execution_plan_snapshot, parent_pipeline_snapshot=pipeline_def.get_parent_pipeline_snapshot(), ) tick.update_with_status(ScheduleTickStatus.SUCCESS, run_id=possibly_invalid_pipeline_run.run_id) # If there were errors, inject them into the event log and fail the run if len(errors) > 0: for error in errors: instance.report_engine_event( error.message, possibly_invalid_pipeline_run, EngineEventData.engine_error(error), ) instance.report_run_failed(possibly_invalid_pipeline_run) stream.send( ScheduledExecutionFailed( run_id=possibly_invalid_pipeline_run.run_id, errors=errors)) return # Otherwise the run should be valid so lets launch it # Need an ExternalPipeline to launch so make one here recon_repo = pipeline.get_reconstructable_repository() repo_location = InProcessRepositoryLocation(recon_repo) external_pipeline = repo_location.get_repository( recon_repo.get_definition().name).get_full_external_pipeline( pipeline_def.name) try: launched_run = instance.launch_run( possibly_invalid_pipeline_run.run_id, external_pipeline) except DagsterLaunchFailedError: error = serializable_error_info_from_exc_info(sys.exc_info()) instance.report_engine_event( error.message, possibly_invalid_pipeline_run, EngineEventData.engine_error(error), ) instance.report_run_failed(possibly_invalid_pipeline_run) stream.send( ScheduledExecutionFailed( run_id=possibly_invalid_pipeline_run.run_id, errors=[error])) return stream.send(ScheduledExecutionSuccess(run_id=launched_run.run_id)) return