def create_execution_metadata(graphql_execution_metadata): return (ExecutionMetadata( graphql_execution_metadata.get('runId'), { t['key']: t['value'] for t in graphql_execution_metadata.get('tags', []) }, ) if graphql_execution_metadata else ExecutionMetadata(run_id=None, tags={}))
def create_execution_metadata(graphql_execution_metadata): return (ExecutionMetadata( run_id=graphql_execution_metadata.get("runId"), tags={ t["key"]: t["value"] for t in graphql_execution_metadata.get("tags", []) }, root_run_id=graphql_execution_metadata.get("rootRunId"), parent_run_id=graphql_execution_metadata.get("parentRunId"), ) if graphql_execution_metadata else ExecutionMetadata(run_id=None, tags={}))
def execute_remote_pipeline_run(host, pipeline_name, environment_dict=None, tags=None, solid_subset=None, mode=None): check.str_param(host, 'host') check.str_param(pipeline_name, 'pipeline_name') environment_dict = check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.opt_dict_param(tags, 'tags', key_type=str, value_type=str) check.opt_nullable_list_param(solid_subset, 'solid_subset', of_type=str) mode = check.opt_str_param(mode, 'mode', DEFAULT_MODE_NAME) selector = ExecutionSelector(pipeline_name, solid_subset) execution_params = ExecutionParams( selector=selector, environment_dict=environment_dict, mode=mode, execution_metadata=ExecutionMetadata(run_id=None, tags=tags or {}), step_keys=None, previous_run_id=None, ) result = execute_query_against_remote( host, START_PIPELINE_EXECUTION_MUTATION, variables=json.dumps( {'executionParams': execution_params.to_graphql_input()}), ) return result
def execution_params_from_pipeline_run(run): check.inst_param(run, 'run', PipelineRun) return ExecutionParams( mode=run.mode, step_keys=run.step_keys_to_execute, environment_dict=run.environment_dict, selector=run.selector, execution_metadata=ExecutionMetadata(run_id=run.run_id, tags=run.tags), )
def create_execution_params(graphene_info, graphql_execution_params): preset_name = graphql_execution_params.get('preset') if preset_name: check.invariant( not graphql_execution_params.get('environmentConfigData'), 'Invalid ExecutionParams. Cannot define environment_dict when using preset', ) check.invariant( not graphql_execution_params.get('mode'), 'Invalid ExecutionParams. Cannot define mode when using preset', ) selector = graphql_execution_params['selector'].to_selector() check.invariant( not selector.solid_subset, 'Invalid ExecutionParams. Cannot define selector.solid_subset when using preset', ) dauphin_pipeline = get_dauphin_pipeline_reference_from_selector(graphene_info, selector) pipeline = dauphin_pipeline.get_dagster_pipeline() if not pipeline.has_preset(preset_name): raise UserFacingGraphQLError( graphene_info.schema.type_named('PresetNotFoundError')( preset=preset_name, selector=selector ) ) preset = pipeline.get_preset(preset_name) return ExecutionParams( selector=ExecutionSelector(selector.name, preset.solid_subset), environment_dict=preset.environment_dict, mode=preset.mode, execution_metadata=ExecutionMetadata(run_id=None, tags={}), step_keys=graphql_execution_params.get('stepKeys'), previous_run_id=graphql_execution_params.get('retryRunId'), ) return ExecutionParams( selector=graphql_execution_params['selector'].to_selector(), environment_dict=graphql_execution_params.get('environmentConfigData'), mode=graphql_execution_params.get('mode'), execution_metadata=create_execution_metadata( graphql_execution_params.get('executionMetadata') ), step_keys=graphql_execution_params.get('stepKeys'), previous_run_id=graphql_execution_params.get('retryRunId'), )
def execution_params_from_pipeline_run(context, run): check.inst_param(run, 'run', PipelineRun) return ExecutionParams( mode=run.mode, step_keys=run.step_keys_to_execute, environment_dict=run.environment_dict, selector=PipelineSelector.legacy(context, run.pipeline_name, run.solid_subset), execution_metadata=ExecutionMetadata( run_id=run.run_id, tags=run.tags, root_run_id=run.root_run_id, parent_run_id=run.parent_run_id, ), )
def start_scheduled_execution(graphene_info, schedule_name): ''' When a scheduler ticks and needs to run for a given schedule, it issues a START_SCHEDULED_EXECUTION mutation with just the schedule name. The mutation is resolved entirely by this method. ''' check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.str_param(schedule_name, 'schedule_name') tick = None try: # We first load the repository and schedule definition to create # and store a ScheduleTick. # If this fails, this error should be sent to the file based scheduler logs. external_repository = graphene_info.context.get_external_repository() repository_name = external_repository.name schedule_def = get_dagster_schedule_def(graphene_info, schedule_name) cron_schedule = "Unknown" if not schedule_def else schedule_def.cron_schedule tick = graphene_info.context.instance.create_schedule_tick( repository_name, ScheduleTickData( schedule_name=schedule_name, cron_schedule=cron_schedule, timestamp=time.time(), status=ScheduleTickStatus.STARTED, ), ) # Run should_execute and halt if it returns False schedule_context = ScheduleExecutionContext( graphene_info.context.instance) with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution should_execute for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): should_execute = schedule_def.should_execute(schedule_context) if not should_execute: # Update tick to skipped state and return tick = tick.with_status(ScheduleTickStatus.SKIPPED) graphene_info.context.instance.update_schedule_tick( repository_name, tick) # Return skipped specific gql response return graphene_info.schema.type_named( 'ScheduledExecutionBlocked' )(message= 'Schedule {schedule_name} did not run because the should_execute did not return' ' True'.format(schedule_name=schedule_name)) errors = [] environment_dict = {} schedule_tags = {} try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of environment_dict_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): environment_dict = schedule_def.get_environment_dict( schedule_context) except DagsterUserCodeExecutionError as exc: error_data = serializable_error_info_from_exc_info(sys.exc_info()) errors.append(error_data) try: with user_code_error_boundary( ScheduleExecutionError, lambda: 'Error occurred during the execution of tags_fn for schedule ' '{schedule_name}'.format(schedule_name=schedule_def.name), ): schedule_tags = schedule_def.get_tags(schedule_context) except DagsterUserCodeExecutionError: error_data = serializable_error_info_from_exc_info(sys.exc_info()) errors.append(error_data) external_pipeline = get_external_pipeline_or_raise( graphene_info, schedule_def.selector.name, schedule_def.selector.solid_subset) pipeline_tags = external_pipeline.tags or {} check_tags(pipeline_tags, 'pipeline_tags') tags = merge_dicts(pipeline_tags, schedule_tags) selector = schedule_def.selector mode = schedule_def.mode execution_params = ExecutionParams( selector=selector, environment_dict=environment_dict, mode=mode, execution_metadata=ExecutionMetadata(tags=tags, run_id=None), step_keys=None, ) run, result = _execute_schedule(graphene_info, external_pipeline, execution_params, errors) graphene_info.context.instance.update_schedule_tick( repository_name, tick.with_status(ScheduleTickStatus.SUCCESS, run_id=run.run_id), ) return result except Exception as exc: # pylint: disable=broad-except error_data = serializable_error_info_from_exc_info(sys.exc_info()) if tick: graphene_info.context.instance.update_schedule_tick( repository_name, tick.with_status(ScheduleTickStatus.FAILURE, error=error_data), ) raise exc