class DauphinWrappingConfigType(dauphin.Interface): class Meta: name = 'WrappingConfigType' of_type = dauphin.Field(dauphin.NonNull(DauphinConfigType))
class DauphinEnvironmentSchema(dauphin.ObjectType): def __init__(self, environment_schema, dagster_pipeline): from dagster.core.definitions.environment_schema import EnvironmentSchema from dagster.core.definitions.pipeline import PipelineDefinition self._environment_schema = check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) self._dagster_pipeline = check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) class Meta(object): name = 'EnvironmentSchema' description = '''The environment schema represents the all the config type information given a certain execution selection and mode of execution of that selection. All config interactions (e.g. checking config validity, fetching all config types, fetching in a particular config type) should be done through this type ''' rootEnvironmentType = dauphin.Field( dauphin.NonNull('ConfigType'), description= '''Fetch the root environment type. Concretely this is the type that is in scope at the root of configuration document for a particular execution selection. It is the type that is in scope initially with a blank config editor.''', ) allConfigTypes = dauphin.Field( dauphin.non_null_list('ConfigType'), description= '''Fetch all the named config types that are in the schema. Useful for things like a type browser UI, or for fetching all the types are in the scope of a document so that the index can be built for the autocompleting editor. ''', ) isEnvironmentConfigValid = dauphin.Field( dauphin.NonNull('PipelineConfigValidationResult'), args={ 'environmentConfigData': dauphin.Argument('EnvironmentConfigData') }, description= '''Parse a particular environment config result. The return value either indicates that the validation succeeded by returning `PipelineConfigValidationValid` or that there are configuration errors by returning `PipelineConfigValidationInvalid' which containers a list errors so that can be rendered for the user''', ) def resolve_allConfigTypes(self, _graphene_info): return sorted( list( map( lambda ct: to_dauphin_config_type( self._dagster_pipeline.get_config_schema_snapshot(), ct .key), self._environment_schema.all_config_types(), )), key=lambda ct: ct.key, ) def resolve_rootEnvironmentType(self, _graphene_info): return to_dauphin_config_type( self._dagster_pipeline.get_config_schema_snapshot(), self._environment_schema.environment_type.key, ) def resolve_isEnvironmentConfigValid(self, graphene_info, **kwargs): return resolve_is_environment_config_valid( graphene_info, self._environment_schema, self._dagster_pipeline, kwargs.get('environmentConfigData', {}), )
class DauphinStepEvent(dauphin.Interface): class Meta(object): name = 'StepEvent' step = dauphin.Field('ExecutionStep')
class DauphinStepEvent(dauphin.Interface): class Meta(object): name = "StepEvent" stepKey = dauphin.Field(dauphin.String) solidHandleID = dauphin.Field(dauphin.String)
class DauphinCancelPipelineExecutionSuccess(dauphin.ObjectType): class Meta(object): name = 'CancelPipelineExecutionSuccess' run = dauphin.Field(dauphin.NonNull('PipelineRun'))
class DauphinStartPipelineExecutionSuccess(dauphin.ObjectType): class Meta: name = 'StartPipelineExecutionSuccess' run = dauphin.Field(dauphin.NonNull('PipelineRun'))
class DauphinPipelineRun(dauphin.ObjectType): class Meta(object): name = "PipelineRun" runId = dauphin.NonNull(dauphin.String) # Nullable because of historical runs pipelineSnapshotId = dauphin.String() status = dauphin.NonNull("PipelineRunStatus") pipeline = dauphin.NonNull("PipelineReference") pipelineName = dauphin.NonNull(dauphin.String) solidSelection = dauphin.List(dauphin.NonNull(dauphin.String)) stats = dauphin.NonNull("PipelineRunStatsOrError") stepStats = dauphin.non_null_list("PipelineRunStepStats") computeLogs = dauphin.Field( dauphin.NonNull("ComputeLogs"), stepKey=dauphin.Argument(dauphin.NonNull(dauphin.String)), description=""" Compute logs are the stdout/stderr logs for a given solid step computation """, ) executionPlan = dauphin.Field("ExecutionPlan") stepKeysToExecute = dauphin.List(dauphin.NonNull(dauphin.String)) runConfigYaml = dauphin.NonNull(dauphin.String) mode = dauphin.NonNull(dauphin.String) tags = dauphin.non_null_list("PipelineTag") rootRunId = dauphin.Field(dauphin.String) parentRunId = dauphin.Field(dauphin.String) canTerminate = dauphin.NonNull(dauphin.Boolean) assets = dauphin.non_null_list("Asset") def __init__(self, pipeline_run): super(DauphinPipelineRun, self).__init__(runId=pipeline_run.run_id, status=pipeline_run.status, mode=pipeline_run.mode) self._pipeline_run = check.inst_param(pipeline_run, "pipeline_run", PipelineRun) def resolve_pipeline(self, graphene_info): return get_pipeline_reference_or_raise( graphene_info, self._pipeline_run, ) def resolve_pipelineName(self, _graphene_info): return self._pipeline_run.pipeline_name def resolve_solidSelection(self, _graphene_info): return self._pipeline_run.solid_selection def resolve_pipelineSnapshotId(self, _): return self._pipeline_run.pipeline_snapshot_id def resolve_stats(self, graphene_info): return get_stats(graphene_info, self.run_id) def resolve_stepStats(self, graphene_info): return get_step_stats(graphene_info, self.run_id) def resolve_computeLogs(self, graphene_info, stepKey): return graphene_info.schema.type_named("ComputeLogs")( runId=self.run_id, stepKey=stepKey) def resolve_executionPlan(self, graphene_info): if not (self._pipeline_run.execution_plan_snapshot_id and self._pipeline_run.pipeline_snapshot_id): return None from .execution import DauphinExecutionPlan instance = graphene_info.context.instance historical_pipeline = instance.get_historical_pipeline( self._pipeline_run.pipeline_snapshot_id) execution_plan_snapshot = instance.get_execution_plan_snapshot( self._pipeline_run.execution_plan_snapshot_id) return (DauphinExecutionPlan( ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot, represented_pipeline=historical_pipeline, )) if execution_plan_snapshot and historical_pipeline else None) def resolve_stepKeysToExecute(self, _): return self._pipeline_run.step_keys_to_execute def resolve_runConfigYaml(self, _graphene_info): return yaml.dump(self._pipeline_run.run_config, default_flow_style=False) def resolve_tags(self, graphene_info): return [ graphene_info.schema.type_named("PipelineTag")(key=key, value=value) for key, value in self._pipeline_run.tags.items() if get_tag_type(key) != TagType.HIDDEN ] def resolve_rootRunId(self, _): return self._pipeline_run.root_run_id def resolve_parentRunId(self, _): return self._pipeline_run.parent_run_id @property def run_id(self): return self.runId def resolve_canTerminate(self, graphene_info): # short circuit if the pipeline run is in a terminal state if self._pipeline_run.is_finished: return False return graphene_info.context.instance.run_launcher.can_terminate( self.run_id) def resolve_assets(self, graphene_info): return get_assets_for_run_id(graphene_info, self.run_id)
class DauphinQuery(dauphin.ObjectType): class Meta: name = 'Query' version = dauphin.NonNull(dauphin.String) pipelineOrError = dauphin.Field( dauphin.NonNull('PipelineOrError'), params=dauphin.NonNull('ExecutionSelector') ) pipeline = dauphin.Field( dauphin.NonNull('Pipeline'), params=dauphin.NonNull('ExecutionSelector') ) pipelinesOrError = dauphin.NonNull('PipelinesOrError') pipelines = dauphin.Field(dauphin.NonNull('PipelineConnection')) configTypeOrError = dauphin.Field( dauphin.NonNull('ConfigTypeOrError'), pipelineName=dauphin.Argument(dauphin.NonNull(dauphin.String)), configTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) runtimeTypeOrError = dauphin.Field( dauphin.NonNull('RuntimeTypeOrError'), pipelineName=dauphin.Argument(dauphin.NonNull(dauphin.String)), runtimeTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) pipelineRuns = dauphin.non_null_list('PipelineRun') pipelineRunOrError = dauphin.Field( dauphin.NonNull('PipelineRunOrError'), runId=dauphin.NonNull(dauphin.ID) ) isPipelineConfigValid = dauphin.Field( dauphin.NonNull('PipelineConfigValidationResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'config': dauphin.Argument('PipelineConfig'), }, ) executionPlan = dauphin.Field( dauphin.NonNull('ExecutionPlanResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'config': dauphin.Argument('PipelineConfig'), }, ) presetsForPipeline = dauphin.Field( dauphin.List(dauphin.NonNull('PipelinePreset')), args={'pipelineName': dauphin.Argument(dauphin.NonNull('String'))}, ) def resolve_configTypeOrError(self, graphene_info, **kwargs): return get_config_type(graphene_info, kwargs['pipelineName'], kwargs['configTypeName']) def resolve_runtimeTypeOrError(self, graphene_info, **kwargs): return get_runtime_type(graphene_info, kwargs['pipelineName'], kwargs['runtimeTypeName']) def resolve_version(self, graphene_info): return graphene_info.context.version def resolve_pipelineOrError(self, graphene_info, **kwargs): return get_pipeline(graphene_info, kwargs['params'].to_selector()) def resolve_pipeline(self, graphene_info, **kwargs): return get_pipeline_or_raise(graphene_info, kwargs['params'].to_selector()) def resolve_pipelinesOrError(self, graphene_info): return get_pipelines(graphene_info) def resolve_pipelines(self, graphene_info): return get_pipelines_or_raise(graphene_info) def resolve_pipelineRuns(self, graphene_info): return get_runs(graphene_info) def resolve_pipelineRunOrError(self, graphene_info, runId): return get_run(graphene_info, runId) def resolve_isPipelineConfigValid(self, graphene_info, pipeline, config): return validate_pipeline_config(graphene_info, pipeline.to_selector(), config) def resolve_executionPlan(self, graphene_info, pipeline, config): return get_execution_plan(graphene_info, pipeline.to_selector(), config) def resolve_presetsForPipeline(self, graphene_info, pipelineName): return get_pipeline_presets(graphene_info, pipelineName)
class DauphinSensor(dauphin.ObjectType): class Meta: name = "Sensor" id = dauphin.NonNull(dauphin.ID) name = dauphin.NonNull(dauphin.String) pipelineName = dauphin.NonNull(dauphin.String) solidSelection = dauphin.List(dauphin.String) mode = dauphin.NonNull(dauphin.String) status = dauphin.NonNull("JobStatus") runs = dauphin.Field(dauphin.non_null_list("PipelineRun"), limit=dauphin.Int()) runsCount = dauphin.NonNull(dauphin.Int) ticks = dauphin.Field(dauphin.non_null_list("JobTick"), limit=dauphin.Int()) def resolve_id(self, _): return "%s:%s" % (self.name, self.pipelineName) def __init__(self, graphene_info, external_sensor): self._external_sensor = check.inst_param(external_sensor, "external_sensor", ExternalSensor) self._sensor_state = graphene_info.context.instance.get_job_state( self._external_sensor.get_external_origin_id()) if not self._sensor_state: # Also include a SensorState for a stopped sensor that may not # have a stored database row yet self._sensor_state = self._external_sensor.get_default_job_state() super(DauphinSensor, self).__init__( name=external_sensor.name, pipelineName=external_sensor.pipeline_name, solidSelection=external_sensor.solid_selection, mode=external_sensor.mode, ) def resolve_status(self, _graphene_info): return self._sensor_state.status def resolve_runs(self, graphene_info, **kwargs): return [ graphene_info.schema.type_named("PipelineRun")(r) for r in graphene_info.context.instance.get_runs( filters=PipelineRunsFilter.for_sensor(self._external_sensor), limit=kwargs.get("limit"), ) ] def resolve_runsCount(self, graphene_info): return graphene_info.context.instance.get_runs_count( filters=PipelineRunsFilter.for_sensor(self._external_sensor)) def resolve_ticks(self, graphene_info, limit=None): ticks = graphene_info.context.instance.get_job_ticks( self._external_sensor.get_external_origin_id()) if limit: ticks = ticks[:limit] return [ graphene_info.schema.type_named("JobTick")(graphene_info, tick) for tick in ticks ]
class DauphinRuntimeMismatchConfigError(dauphin.ObjectType): class Meta(object): name = "RuntimeMismatchConfigError" interfaces = (DauphinPipelineConfigValidationError, ) value_rep = dauphin.Field(dauphin.String)
class DauphinLaunchPipelineRunSuccess(dauphin.ObjectType): class Meta(object): name = "LaunchPipelineRunSuccess" run = dauphin.Field(dauphin.NonNull("PipelineRun"))
class DauphinPipeline(dauphin.ObjectType): class Meta: name = 'Pipeline' interfaces = [DauphinSolidContainer] name = dauphin.NonNull(dauphin.String) description = dauphin.String() solids = dauphin.non_null_list('Solid') environment_type = dauphin.Field( dauphin.NonNull('ConfigType'), mode=dauphin.String(required=False) ) config_types = dauphin.Field( dauphin.non_null_list('ConfigType'), mode=dauphin.String(required=False) ) runtime_types = dauphin.non_null_list('RuntimeType') runs = dauphin.non_null_list('PipelineRun') modes = dauphin.non_null_list('Mode') solid_handles = dauphin.non_null_list('SolidHandle') presets = dauphin.non_null_list('PipelinePreset') def __init__(self, pipeline): super(DauphinPipeline, self).__init__(name=pipeline.name, description=pipeline.description) self._pipeline = check.inst_param(pipeline, 'pipeline', PipelineDefinition) def resolve_solids(self, _graphene_info): return build_dauphin_solids(self._pipeline) def resolve_environment_type(self, _graphene_info, mode=None): return to_dauphin_config_type(create_environment_type(self._pipeline, mode)) def resolve_config_types(self, _graphene_info, mode=None): environment_schema = create_environment_schema(self._pipeline, mode) return sorted( list(map(to_dauphin_config_type, environment_schema.all_config_types())), key=lambda config_type: config_type.key, ) def resolve_runtime_types(self, _graphene_info): return sorted( list( map( to_dauphin_runtime_type, [t for t in self._pipeline.all_runtime_types() if t.name], ) ), key=lambda config_type: config_type.name, ) def resolve_runs(self, graphene_info): return [ graphene_info.schema.type_named('PipelineRun')(r) for r in graphene_info.context.pipeline_runs.all_runs_for_pipeline(self._pipeline.name) ] def get_dagster_pipeline(self): return self._pipeline def get_type(self, _graphene_info, typeName): if self._pipeline.has_config_type(typeName): return to_dauphin_config_type(self._pipeline.config_type_named(typeName)) elif self._pipeline.has_runtime_type(typeName): return to_dauphin_runtime_type(self._pipeline.runtime_type_named(typeName)) else: check.failed('Not a config type or runtime type') def resolve_modes(self, graphene_info): return [ graphene_info.schema.type_named('Mode')(mode_definition) for mode_definition in sorted( self._pipeline.mode_definitions, key=lambda item: item.name ) ] def resolve_solid_handles(self, _graphene_info): return sorted( build_dauphin_solid_handles(self._pipeline), key=lambda item: str(item.handleID) ) def resolve_presets(self, _graphene_info): return [ DauphinPipelinePreset(preset, self._pipeline.name) for preset in sorted(self._pipeline.get_presets(), key=lambda item: item.name) ]
class DauphinWrappingDagsterType(dauphin.Interface): class Meta(object): name = 'WrappingDagsterType' of_type = dauphin.Field(dauphin.NonNull(DauphinDagsterType))
class DauphinRunningSchedule(dauphin.ObjectType): class Meta: name = 'RunningSchedule' id = dauphin.NonNull(dauphin.String) schedule_definition = dauphin.NonNull('ScheduleDefinition') python_path = dauphin.Field(dauphin.String) repository_path = dauphin.Field(dauphin.String) status = dauphin.NonNull('ScheduleStatus') runs = dauphin.Field(dauphin.non_null_list('PipelineRun'), limit=dauphin.Int()) runs_count = dauphin.NonNull(dauphin.Int) attempts = dauphin.Field(dauphin.non_null_list('ScheduleAttempt'), limit=dauphin.Int()) logs_path = dauphin.NonNull(dauphin.String) def __init__(self, graphene_info, schedule): self._schedule = check.inst_param(schedule, 'schedule', Schedule) super(DauphinRunningSchedule, self).__init__( id=schedule.schedule_id, schedule_definition=graphene_info.schema.type_named( 'ScheduleDefinition')(get_dagster_schedule_def( graphene_info, schedule.name)), status=schedule.status, python_path=schedule.python_path, repository_path=schedule.repository_path, ) def resolve_attempts(self, graphene_info, **kwargs): limit = kwargs.get('limit') scheduler = graphene_info.context.get_scheduler() log_dir = scheduler.log_path_for_schedule(self._schedule.name) results = glob.glob(os.path.join(log_dir, "*.result")) if limit is None: limit = len(results) latest_results = heapq.nlargest(limit, results, key=os.path.getctime) attempts = [] for result_path in latest_results: with open(result_path, 'r') as f: line = f.readline() if not line: continue # File is empty start_scheduled_execution_response = json.loads(line) json_result = start_scheduled_execution_response['data'][ 'startScheduledExecution'] typename = json_result['__typename'] if typename == 'StartPipelineExecutionSuccess': status = DauphinScheduleAttemptStatus.SUCCESS elif typename == 'ScheduleExecutionBlocked': status = DauphinScheduleAttemptStatus.SKIPPED else: status = DauphinScheduleAttemptStatus.ERROR run = None if typename == 'StartPipelineExecutionSuccess': run_id = json_result['run']['runId'] run = graphene_info.schema.type_named('PipelineRun')( graphene_info.context.instance.get_run_by_id(run_id)) attempts.append( graphene_info.schema.type_named('ScheduleAttempt')( time=os.path.getctime(result_path), json_result=json.dumps(json_result), status=status, run=run, )) return attempts def resolve_logs_path(self, graphene_info): scheduler = graphene_info.context.get_scheduler() return scheduler.log_path_for_schedule(self._schedule.name) def resolve_runs(self, graphene_info, **kwargs): return [ graphene_info.schema.type_named('PipelineRun')(r) for r in graphene_info.context.instance. get_runs_with_matching_tags([("dagster/schedule_id", self._schedule.schedule_id)], limit=kwargs.get('limit')) ] def resolve_runs_count(self, graphene_info): return graphene_info.context.instance.get_run_count_with_matching_tags( [("dagster/schedule_id", self._schedule.schedule_id)])
class DauphinPipelineConfigValidationValid(dauphin.ObjectType): class Meta: name = 'PipelineConfigValidationValid' pipeline = dauphin.Field(dauphin.NonNull('Pipeline'))
class DauphinPipelineRun(dauphin.ObjectType): class Meta: name = 'PipelineRun' runId = dauphin.NonNull(dauphin.String) status = dauphin.NonNull('PipelineRunStatus') pipeline = dauphin.NonNull('PipelineReference') stats = dauphin.NonNull('PipelineRunStatsSnapshot') logs = dauphin.NonNull('LogMessageConnection') computeLogs = dauphin.Field( dauphin.NonNull('ComputeLogs'), stepKey=dauphin.Argument(dauphin.NonNull(dauphin.String)), description=''' Compute logs are the stdout/stderr logs for a given solid step computation ''', ) executionPlan = dauphin.Field('ExecutionPlan') stepKeysToExecute = dauphin.List(dauphin.NonNull(dauphin.String)) environmentConfigYaml = dauphin.NonNull(dauphin.String) mode = dauphin.NonNull(dauphin.String) tags = dauphin.non_null_list('PipelineTag') canCancel = dauphin.NonNull(dauphin.Boolean) def __init__(self, pipeline_run): super(DauphinPipelineRun, self).__init__(runId=pipeline_run.run_id, status=pipeline_run.status, mode=pipeline_run.mode) self._pipeline_run = check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) def resolve_pipeline(self, graphene_info): return get_pipeline_reference_or_raise(graphene_info, self._pipeline_run.selector) def resolve_logs(self, graphene_info): return graphene_info.schema.type_named('LogMessageConnection')( self._pipeline_run) def resolve_stats(self, graphene_info): stats = graphene_info.context.instance.get_run_stats(self.run_id) return graphene_info.schema.type_named('PipelineRunStatsSnapshot')( stats) def resolve_computeLogs(self, graphene_info, stepKey): return graphene_info.schema.type_named('ComputeLogs')( runId=self.run_id, stepKey=stepKey) def resolve_executionPlan(self, graphene_info): pipeline = self.resolve_pipeline(graphene_info) if isinstance(pipeline, DauphinPipeline): execution_plan = create_execution_plan( pipeline.get_dagster_pipeline(), self._pipeline_run.environment_dict, RunConfig(mode=self._pipeline_run.mode), ) return graphene_info.schema.type_named('ExecutionPlan')( pipeline, execution_plan) else: return None def resolve_stepKeysToExecute(self, _): return self._pipeline_run.step_keys_to_execute def resolve_environmentConfigYaml(self, _graphene_info): return yaml.dump(self._pipeline_run.environment_dict, default_flow_style=False) def resolve_tags(self, graphene_info): return [ graphene_info.schema.type_named('PipelineTag')(key=key, value=value) for key, value in self._pipeline_run.tags.items() ] @property def run_id(self): return self.runId def resolve_canCancel(self, graphene_info): return graphene_info.context.execution_manager.can_terminate( self.run_id)
class DauphinPipelineConfigValidationInvalid(dauphin.ObjectType): class Meta: name = 'PipelineConfigValidationInvalid' pipeline = dauphin.Field(dauphin.NonNull('Pipeline')) errors = dauphin.non_null_list('PipelineConfigValidationError')
class DauphinTerminatePipelineExecutionSuccess(dauphin.ObjectType): class Meta(object): name = "TerminatePipelineExecutionSuccess" run = dauphin.Field(dauphin.NonNull("PipelineRun"))
class DauphinJobState(dauphin.ObjectType): class Meta: name = "JobState" id = dauphin.NonNull(dauphin.ID) name = dauphin.NonNull(dauphin.String) jobType = dauphin.NonNull("JobType") status = dauphin.NonNull("JobStatus") repositoryOrigin = dauphin.NonNull("RepositoryOrigin") jobSpecificData = dauphin.Field("JobSpecificData") runs = dauphin.Field(dauphin.non_null_list("PipelineRun"), limit=dauphin.Int()) runsCount = dauphin.NonNull(dauphin.Int) ticks = dauphin.Field(dauphin.non_null_list("JobTick"), limit=dauphin.Int()) runningCount = dauphin.NonNull(dauphin.Int) # remove with cron scheduler def __init__(self, job_state): self._job_state = check.inst_param(job_state, "job_state", JobState) super(DauphinJobState, self).__init__( id=job_state.job_origin_id, name=job_state.name, jobType=job_state.job_type, status=job_state.status, ) def resolve_repositoryOrigin(self, graphene_info): origin = self._job_state.origin.external_repository_origin return graphene_info.schema.type_named("RepositoryOrigin")(origin) def resolve_jobSpecificData(self, graphene_info): if not self._job_state.job_specific_data: return None if self._job_state.job_type == JobType.SENSOR: return graphene_info.schema.type_named("SensorJobData")( self._job_state.job_specific_data ) if self._job_state.job_type == JobType.SCHEDULE: return graphene_info.schema.type_named("ScheduleJobData")( self._job_state.job_specific_data ) return None def resolve_runs(self, graphene_info, **kwargs): if self._job_state.job_type == JobType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._job_state) else: filters = PipelineRunsFilter.for_schedule(self._job_state) return [ graphene_info.schema.type_named("PipelineRun")(r) for r in graphene_info.context.instance.get_runs( filters=filters, limit=kwargs.get("limit"), ) ] def resolve_runsCount(self, graphene_info): if self._job_state.job_type == JobType.SENSOR: filters = PipelineRunsFilter.for_sensor(self._job_state) else: filters = PipelineRunsFilter.for_schedule(self._job_state) return graphene_info.context.instance.get_runs_count(filters=filters) def resolve_ticks(self, graphene_info, limit=None): ticks = graphene_info.context.instance.get_job_ticks(self._job_state.job_origin_id) if limit: ticks = ticks[:limit] return [graphene_info.schema.type_named("JobTick")(graphene_info, tick) for tick in ticks] def resolve_runningCount(self, graphene_info): if self._job_state.job_type == JobType.SENSOR: return 1 if self._job_state.status == JobStatus.RUNNING else 0 else: return graphene_info.context.instance.running_schedule_count( self._job_state.job_origin_id )
class DauphinRetries(dauphin.InputObjectType): class Meta(object): name = "Retries" mode = dauphin.Field(dauphin.String) retries_previous_attempts = dauphin.List(DauphinRetriesPreviousAttempts)
class DauphinPipelineFailureEvent(dauphin.ObjectType): class Meta(object): name = "PipelineFailureEvent" interfaces = (DauphinMessageEvent, DauphinPipelineEvent) error = dauphin.Field("PythonError")
class DauphinQuery(dauphin.ObjectType): class Meta(object): name = "Query" version = dauphin.NonNull(dauphin.String) repositoriesOrError = dauphin.NonNull("RepositoriesOrError") repositoryOrError = dauphin.Field( dauphin.NonNull("RepositoryOrError"), repositorySelector=dauphin.NonNull("RepositorySelector"), ) pipelineOrError = dauphin.Field(dauphin.NonNull("PipelineOrError"), params=dauphin.NonNull("PipelineSelector")) pipelineSnapshotOrError = dauphin.Field( dauphin.NonNull("PipelineSnapshotOrError"), snapshotId=dauphin.String(), activePipelineSelector=dauphin.Argument("PipelineSelector"), ) scheduler = dauphin.Field(dauphin.NonNull("SchedulerOrError")) scheduleDefinitionOrError = dauphin.Field( dauphin.NonNull("ScheduleDefinitionOrError"), schedule_selector=dauphin.NonNull("ScheduleSelector"), ) scheduleDefinitionsOrError = dauphin.Field( dauphin.NonNull("ScheduleDefinitionsOrError"), repositorySelector=dauphin.NonNull("RepositorySelector"), ) scheduleStatesOrError = dauphin.Field( dauphin.NonNull("ScheduleStatesOrError"), repositorySelector=dauphin.Argument("RepositorySelector"), withNoScheduleDefinition=dauphin.Boolean(), ) partitionSetsOrError = dauphin.Field( dauphin.NonNull("PartitionSetsOrError"), repositorySelector=dauphin.NonNull("RepositorySelector"), pipelineName=dauphin.NonNull(dauphin.String), ) partitionSetOrError = dauphin.Field( dauphin.NonNull("PartitionSetOrError"), repositorySelector=dauphin.NonNull("RepositorySelector"), partitionSetName=dauphin.String(), ) pipelineRunsOrError = dauphin.Field( dauphin.NonNull("PipelineRunsOrError"), filter=dauphin.Argument("PipelineRunsFilter"), cursor=dauphin.String(), limit=dauphin.Int(), ) pipelineRunOrError = dauphin.Field(dauphin.NonNull("PipelineRunOrError"), runId=dauphin.NonNull(dauphin.ID)) pipelineRunTags = dauphin.non_null_list("PipelineTagAndValues") runGroupOrError = dauphin.Field(dauphin.NonNull("RunGroupOrError"), runId=dauphin.NonNull(dauphin.ID)) runGroupsOrError = dauphin.Field( dauphin.NonNull("RunGroupsOrError"), filter=dauphin.Argument("PipelineRunsFilter"), cursor=dauphin.String(), limit=dauphin.Int(), ) isPipelineConfigValid = dauphin.Field( dauphin.NonNull("PipelineConfigValidationResult"), args={ "pipeline": dauphin.Argument(dauphin.NonNull("PipelineSelector")), "runConfigData": dauphin.Argument("RunConfigData"), "mode": dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) executionPlanOrError = dauphin.Field( dauphin.NonNull("ExecutionPlanOrError"), args={ "pipeline": dauphin.Argument(dauphin.NonNull("PipelineSelector")), "runConfigData": dauphin.Argument("RunConfigData"), "mode": dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) runConfigSchemaOrError = dauphin.Field( dauphin.NonNull("RunConfigSchemaOrError"), args={ "selector": dauphin.Argument(dauphin.NonNull("PipelineSelector")), "mode": dauphin.Argument(dauphin.String), }, description= """Fetch an environment schema given an execution selection and a mode. See the descripton on RunConfigSchema for more information.""", ) instance = dauphin.NonNull("Instance") assetsOrError = dauphin.Field(dauphin.NonNull("AssetsOrError")) assetOrError = dauphin.Field( dauphin.NonNull("AssetOrError"), assetKey=dauphin.Argument(dauphin.NonNull("AssetKeyInput")), ) def resolve_repositoriesOrError(self, graphene_info): return fetch_repositories(graphene_info) def resolve_repositoryOrError(self, graphene_info, **kwargs): return fetch_repository( graphene_info, RepositorySelector.from_graphql_input( kwargs.get("repositorySelector")), ) def resolve_pipelineSnapshotOrError(self, graphene_info, **kwargs): snapshot_id_arg = kwargs.get("snapshotId") pipeline_selector_arg = kwargs.get("activePipelineSelector") check.invariant( not (snapshot_id_arg and pipeline_selector_arg), "Must only pass one of snapshotId or activePipelineSelector", ) check.invariant( snapshot_id_arg or pipeline_selector_arg, "Must set one of snapshotId or activePipelineSelector", ) if pipeline_selector_arg: pipeline_selector = pipeline_selector_from_graphql( graphene_info.context, kwargs["activePipelineSelector"]) return get_pipeline_snapshot_or_error_from_pipeline_selector( graphene_info, pipeline_selector) else: return get_pipeline_snapshot_or_error_from_snapshot_id( graphene_info, snapshot_id_arg) def resolve_version(self, graphene_info): return graphene_info.context.version def resolve_scheduler(self, graphene_info): return get_scheduler_or_error(graphene_info) def resolve_scheduleDefinitionOrError(self, graphene_info, schedule_selector): return get_schedule_definition_or_error( graphene_info, ScheduleSelector.from_graphql_input(schedule_selector)) def resolve_scheduleDefinitionsOrError(self, graphene_info, **kwargs): return get_schedule_definitions_or_error( graphene_info, RepositorySelector.from_graphql_input( kwargs.get("repositorySelector"))) def resolve_scheduleStatesOrError(self, graphene_info, **kwargs): return get_schedule_states_or_error( graphene_info, RepositorySelector.from_graphql_input(kwargs["repositorySelector"]) if kwargs.get("repositorySelector") else None, kwargs.get("withNoScheduleDefinition"), ) def resolve_pipelineOrError(self, graphene_info, **kwargs): return get_pipeline_or_error( graphene_info, pipeline_selector_from_graphql(graphene_info.context, kwargs["params"]), ) def resolve_pipelineRunsOrError(self, graphene_info, **kwargs): filters = kwargs.get("filter") if filters is not None: filters = filters.to_selector() return graphene_info.schema.type_named("PipelineRuns")( results=get_runs(graphene_info, filters, kwargs.get("cursor"), kwargs.get("limit"))) def resolve_pipelineRunOrError(self, graphene_info, runId): return get_run_by_id(graphene_info, runId) def resolve_runGroupsOrError(self, graphene_info, **kwargs): filters = kwargs.get("filter") if filters is not None: filters = filters.to_selector() return graphene_info.schema.type_named("RunGroupsOrError")( results=get_run_groups(graphene_info, filters, kwargs.get( "cursor"), kwargs.get("limit"))) def resolve_partitionSetsOrError(self, graphene_info, **kwargs): return get_partition_sets_or_error( graphene_info, RepositorySelector.from_graphql_input( kwargs.get("repositorySelector")), kwargs.get("pipelineName"), ) def resolve_partitionSetOrError(self, graphene_info, **kwargs): return get_partition_set( graphene_info, RepositorySelector.from_graphql_input( kwargs.get("repositorySelector")), kwargs.get("partitionSetName"), ) def resolve_pipelineRunTags(self, graphene_info): return get_run_tags(graphene_info) def resolve_runGroupOrError(self, graphene_info, runId): return get_run_group(graphene_info, runId) def resolve_isPipelineConfigValid(self, graphene_info, pipeline, **kwargs): return validate_pipeline_config( graphene_info, pipeline_selector_from_graphql(graphene_info.context, pipeline), kwargs.get("runConfigData"), kwargs.get("mode"), ) def resolve_executionPlanOrError(self, graphene_info, pipeline, **kwargs): return get_execution_plan( graphene_info, pipeline_selector_from_graphql(graphene_info.context, pipeline), kwargs.get("runConfigData"), kwargs.get("mode"), ) def resolve_runConfigSchemaOrError(self, graphene_info, **kwargs): return resolve_run_config_schema_or_error( graphene_info, pipeline_selector_from_graphql(graphene_info.context, kwargs["selector"]), kwargs.get("mode"), ) def resolve_instance(self, graphene_info): return graphene_info.schema.type_named("Instance")( graphene_info.context.instance) def resolve_assetsOrError(self, graphene_info): return get_assets(graphene_info) def resolve_assetOrError(self, graphene_info, **kwargs): return get_asset(graphene_info, AssetKey.from_graphql_input(kwargs["assetKey"]))
class DauphinQuery(dauphin.ObjectType): class Meta: name = 'Query' version = dauphin.NonNull(dauphin.String) reloadSupported = dauphin.NonNull(dauphin.Boolean) pipelineOrError = dauphin.Field( dauphin.NonNull('PipelineOrError'), params=dauphin.NonNull('ExecutionSelector') ) pipeline = dauphin.Field( dauphin.NonNull('Pipeline'), params=dauphin.NonNull('ExecutionSelector') ) pipelinesOrError = dauphin.NonNull('PipelinesOrError') pipelines = dauphin.Field(dauphin.NonNull('PipelineConnection')) configTypeOrError = dauphin.Field( dauphin.NonNull('ConfigTypeOrError'), pipelineName=dauphin.Argument(dauphin.NonNull(dauphin.String)), configTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), mode=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) runtimeTypeOrError = dauphin.Field( dauphin.NonNull('RuntimeTypeOrError'), pipelineName=dauphin.Argument(dauphin.NonNull(dauphin.String)), runtimeTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) scheduler = dauphin.Field(dauphin.NonNull('SchedulerOrError')) pipelineRunsOrError = dauphin.Field( dauphin.NonNull('PipelineRunsOrError'), filter=dauphin.Argument(dauphin.NonNull('PipelineRunsFilter')), cursor=dauphin.String(), limit=dauphin.Int(), ) pipelineRunOrError = dauphin.Field( dauphin.NonNull('PipelineRunOrError'), runId=dauphin.NonNull(dauphin.ID) ) pipelineRunTags = dauphin.non_null_list('PipelineTagAndValues') usedSolids = dauphin.Field(dauphin.non_null_list('UsedSolid')) isPipelineConfigValid = dauphin.Field( dauphin.NonNull('PipelineConfigValidationResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'environmentConfigData': dauphin.Argument('EnvironmentConfigData'), 'mode': dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) executionPlan = dauphin.Field( dauphin.NonNull('ExecutionPlanResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'environmentConfigData': dauphin.Argument('EnvironmentConfigData'), 'mode': dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) environmentSchemaOrError = dauphin.Field( dauphin.NonNull('EnvironmentSchemaOrError'), args={ 'selector': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'mode': dauphin.Argument(dauphin.String), }, description='''Fetch an environment schema given an execution selection and a mode. See the descripton on EnvironmentSchema for more information.''', ) instance = dauphin.NonNull('Instance') def resolve_configTypeOrError(self, graphene_info, **kwargs): return get_config_type( graphene_info, kwargs['pipelineName'], kwargs['configTypeName'], kwargs.get('mode') ) def resolve_runtimeTypeOrError(self, graphene_info, **kwargs): return get_runtime_type(graphene_info, kwargs['pipelineName'], kwargs['runtimeTypeName']) def resolve_version(self, graphene_info): return graphene_info.context.version def resolve_reloadSupported(self, graphene_info): return graphene_info.context.reloader.is_reload_supported def resolve_scheduler(self, graphene_info): return get_scheduler_or_error(graphene_info) def resolve_pipelineOrError(self, graphene_info, **kwargs): return get_pipeline_or_error(graphene_info, kwargs['params'].to_selector()) def resolve_pipeline(self, graphene_info, **kwargs): return get_pipeline_or_raise(graphene_info, kwargs['params'].to_selector()) def resolve_pipelinesOrError(self, graphene_info): return get_pipelines_or_error(graphene_info) def resolve_pipelines(self, graphene_info): return get_pipelines_or_raise(graphene_info) def resolve_pipelineRunsOrError(self, graphene_info, **kwargs): filters = kwargs['filter'].to_selector() provided = [ i for i in [filters.run_id, filters.pipeline, filters.tag_key, filters.status] if i ] if len(provided) > 1: return graphene_info.schema.type_named('InvalidPipelineRunsFilterError')( message="You may only provide one of the filter options." ) return graphene_info.schema.type_named('PipelineRuns')( results=get_runs(graphene_info, filters, kwargs.get('cursor'), kwargs.get('limit')) ) def resolve_pipelineRunOrError(self, graphene_info, runId): return get_run(graphene_info, runId) def resolve_pipelineRunTags(self, graphene_info): return get_run_tags(graphene_info) def resolve_usedSolids(self, graphene_info): repository = graphene_info.context.repository_definition inv_by_def_name = defaultdict(list) definitions = [] for pipeline in repository.get_all_pipelines(): for handle in build_dauphin_solid_handles(pipeline): definition = handle.solid.resolve_definition(graphene_info) if definition.name not in inv_by_def_name: definitions.append(definition) inv_by_def_name[definition.name].append( DauphinSolidInvocationSite(pipeline=pipeline, solidHandle=handle) ) return map( lambda d: DauphinUsedSolid( definition=d, invocations=sorted(inv_by_def_name[d.name], key=lambda i: i.solidHandle.handleID), ), sorted(definitions, key=lambda d: d.name), ) def resolve_isPipelineConfigValid(self, graphene_info, pipeline, **kwargs): return validate_pipeline_config( graphene_info, pipeline.to_selector(), kwargs.get('environmentConfigData'), kwargs.get('mode'), ) def resolve_executionPlan(self, graphene_info, pipeline, **kwargs): return get_execution_plan( graphene_info, pipeline.to_selector(), kwargs.get('environmentConfigData'), kwargs.get('mode'), ) def resolve_environmentSchemaOrError(self, graphene_info, **kwargs): return resolve_environment_schema_or_error( graphene_info, kwargs['selector'].to_selector(), kwargs.get('mode') ) def resolve_instance(self, graphene_info): return graphene_info.schema.type_named('Instance')(graphene_info.context.instance)
class DauphinRunConfigSchema(dauphin.ObjectType): def __init__(self, represented_pipeline, mode): self._represented_pipeline = check.inst_param(represented_pipeline, "represented_pipeline", RepresentedPipeline) self._mode = check.str_param(mode, "mode") class Meta(object): name = "RunConfigSchema" description = """The run config schema represents the all the config type information given a certain execution selection and mode of execution of that selection. All config interactions (e.g. checking config validity, fetching all config types, fetching in a particular config type) should be done through this type """ rootConfigType = dauphin.Field( dauphin.NonNull("ConfigType"), description= """Fetch the root environment type. Concretely this is the type that is in scope at the root of configuration document for a particular execution selection. It is the type that is in scope initially with a blank config editor.""", ) allConfigTypes = dauphin.Field( dauphin.non_null_list("ConfigType"), description= """Fetch all the named config types that are in the schema. Useful for things like a type browser UI, or for fetching all the types are in the scope of a document so that the index can be built for the autocompleting editor. """, ) isRunConfigValid = dauphin.Field( dauphin.NonNull("PipelineConfigValidationResult"), args={"runConfigData": dauphin.Argument("RunConfigData")}, description= """Parse a particular environment config result. The return value either indicates that the validation succeeded by returning `PipelineConfigValidationValid` or that there are configuration errors by returning `PipelineConfigValidationInvalid' which containers a list errors so that can be rendered for the user""", ) def resolve_allConfigTypes(self, _graphene_info): return sorted( list( map( lambda key: to_dauphin_config_type( self._represented_pipeline.config_schema_snapshot, key ), self._represented_pipeline.config_schema_snapshot. all_config_keys, )), key=lambda ct: ct.key, ) def resolve_rootConfigType(self, _graphene_info): return to_dauphin_config_type( self._represented_pipeline.config_schema_snapshot, self._represented_pipeline.get_mode_def_snap( self._mode).root_config_key, ) def resolve_isRunConfigValid(self, graphene_info, **kwargs): return resolve_is_run_config_valid( graphene_info, self._represented_pipeline, self._mode, kwargs.get("runConfigData", {}), )
class DauphinQuery(dauphin.ObjectType): class Meta(object): name = 'Query' version = dauphin.NonNull(dauphin.String) reloadSupported = dauphin.NonNull(dauphin.Boolean) pipelineOrError = dauphin.Field( dauphin.NonNull('PipelineOrError'), params=dauphin.NonNull('ExecutionSelector')) pipeline = dauphin.Field(dauphin.NonNull('Pipeline'), params=dauphin.NonNull('ExecutionSelector')) pipelinesOrError = dauphin.NonNull('PipelinesOrError') pipelines = dauphin.Field(dauphin.NonNull('PipelineConnection')) pipelineSnapshot = dauphin.Field( dauphin.NonNull('PipelineSnapshot'), snapshotId=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) runtimeTypeOrError = dauphin.Field( dauphin.NonNull('RuntimeTypeOrError'), pipelineName=dauphin.Argument(dauphin.NonNull(dauphin.String)), runtimeTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) scheduler = dauphin.Field(dauphin.NonNull('SchedulerOrError')) scheduleOrError = dauphin.Field( dauphin.NonNull('ScheduleOrError'), schedule_name=dauphin.NonNull(dauphin.String), limit=dauphin.Int(), ) partitionSetsOrError = dauphin.Field( dauphin.NonNull('PartitionSetsOrError'), pipelineName=dauphin.String()) partitionSetOrError = dauphin.Field(dauphin.NonNull('PartitionSetOrError'), partitionSetName=dauphin.String()) pipelineRunsOrError = dauphin.Field( dauphin.NonNull('PipelineRunsOrError'), filter=dauphin.Argument('PipelineRunsFilter'), cursor=dauphin.String(), limit=dauphin.Int(), ) pipelineRunOrError = dauphin.Field(dauphin.NonNull('PipelineRunOrError'), runId=dauphin.NonNull(dauphin.ID)) pipelineRunTags = dauphin.non_null_list('PipelineTagAndValues') usedSolids = dauphin.Field(dauphin.non_null_list('UsedSolid')) usedSolid = dauphin.Field('UsedSolid', name=dauphin.NonNull(dauphin.String)) isPipelineConfigValid = dauphin.Field( dauphin.NonNull('PipelineConfigValidationResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'environmentConfigData': dauphin.Argument('EnvironmentConfigData'), 'mode': dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) executionPlan = dauphin.Field( dauphin.NonNull('ExecutionPlanResult'), args={ 'pipeline': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'environmentConfigData': dauphin.Argument('EnvironmentConfigData'), 'mode': dauphin.Argument(dauphin.NonNull(dauphin.String)), }, ) environmentSchemaOrError = dauphin.Field( dauphin.NonNull('EnvironmentSchemaOrError'), args={ 'selector': dauphin.Argument(dauphin.NonNull('ExecutionSelector')), 'mode': dauphin.Argument(dauphin.String), }, description= '''Fetch an environment schema given an execution selection and a mode. See the descripton on EnvironmentSchema for more information.''', ) instance = dauphin.NonNull('Instance') def resolve_pipelineSnapshot(self, graphene_info, **kwargs): return get_pipeline_snapshot_or_error(graphene_info, kwargs['snapshotId']) def resolve_runtimeTypeOrError(self, graphene_info, **kwargs): return get_dagster_type(graphene_info, kwargs['pipelineName'], kwargs['runtimeTypeName']) def resolve_version(self, graphene_info): return graphene_info.context.version def resolve_reloadSupported(self, graphene_info): if isinstance(graphene_info.context, DagsterSnapshotGraphQLContext): return False return graphene_info.context.reloader.is_reload_supported def resolve_scheduler(self, graphene_info): return get_scheduler_or_error(graphene_info) def resolve_scheduleOrError(self, graphene_info, schedule_name): return get_schedule_or_error(graphene_info, schedule_name) def resolve_pipelineOrError(self, graphene_info, **kwargs): return get_pipeline_or_error(graphene_info, kwargs['params'].to_selector()) def resolve_pipeline(self, graphene_info, **kwargs): return get_pipeline_or_raise(graphene_info, kwargs['params'].to_selector()) def resolve_pipelinesOrError(self, graphene_info): return get_pipelines_or_error(graphene_info) def resolve_pipelines(self, graphene_info): return get_pipelines_or_raise(graphene_info) def resolve_pipelineRunsOrError(self, graphene_info, **kwargs): filters = kwargs.get('filter') if filters is not None: filters = filters.to_selector() return graphene_info.schema.type_named('PipelineRuns')( results=get_runs(graphene_info, filters, kwargs.get('cursor'), kwargs.get('limit'))) def resolve_pipelineRunOrError(self, graphene_info, runId): return get_run(graphene_info, runId) def resolve_partitionSetsOrError(self, graphene_info, **kwargs): pipeline_name = kwargs.get('pipelineName') return get_partition_sets_or_error(graphene_info, pipeline_name) def resolve_partitionSetOrError(self, graphene_info, partitionSetName): return get_partition_set(graphene_info, partitionSetName) def resolve_pipelineRunTags(self, graphene_info): return get_run_tags(graphene_info) def resolve_usedSolid(self, graphene_info, name): return get_solid(graphene_info, name) def resolve_usedSolids(self, graphene_info): return get_solids(graphene_info) def resolve_isPipelineConfigValid(self, graphene_info, pipeline, **kwargs): return validate_pipeline_config( graphene_info, pipeline.to_selector(), kwargs.get('environmentConfigData'), kwargs.get('mode'), ) def resolve_executionPlan(self, graphene_info, pipeline, **kwargs): return get_execution_plan( graphene_info, pipeline.to_selector(), kwargs.get('environmentConfigData'), kwargs.get('mode'), ) def resolve_environmentSchemaOrError(self, graphene_info, **kwargs): return resolve_environment_schema_or_error( graphene_info, kwargs['selector'].to_selector(), kwargs.get('mode')) def resolve_instance(self, graphene_info): return graphene_info.schema.type_named('Instance')( graphene_info.context.instance)
class DauphinRunningSchedule(dauphin.ObjectType): class Meta(object): name = 'RunningSchedule' schedule_definition = dauphin.NonNull('ScheduleDefinition') python_path = dauphin.Field(dauphin.String) repository_path = dauphin.Field(dauphin.String) status = dauphin.NonNull('ScheduleStatus') runs = dauphin.Field(dauphin.non_null_list('PipelineRun'), limit=dauphin.Int()) runs_count = dauphin.NonNull(dauphin.Int) attempts = dauphin.Field(dauphin.non_null_list('ScheduleAttempt'), limit=dauphin.Int()) attempts_count = dauphin.NonNull(dauphin.Int) logs_path = dauphin.NonNull(dauphin.String) def __init__(self, graphene_info, schedule): self._schedule = check.inst_param(schedule, 'schedule', Schedule) super(DauphinRunningSchedule, self).__init__( schedule_definition=graphene_info.schema.type_named('ScheduleDefinition')( graphene_info=graphene_info, schedule_def=get_dagster_schedule_def(graphene_info, schedule.name), ), status=schedule.status, python_path=schedule.python_path, repository_path=schedule.repository_path, ) def resolve_attempts(self, graphene_info, **kwargs): limit = kwargs.get('limit') results = get_schedule_attempt_filenames(graphene_info, self._schedule.name) if limit is None: limit = len(results) latest_results = heapq.nlargest(limit, results, key=os.path.getctime) attempts = [] for result_path in latest_results: with open(result_path, 'r') as f: line = f.readline() if not line: continue # File is empty start_scheduled_execution_response = json.loads(line) run = None if 'errors' in start_scheduled_execution_response: status = DauphinScheduleAttemptStatus.ERROR json_result = start_scheduled_execution_response['errors'] else: json_result = start_scheduled_execution_response['data'][ 'startScheduledExecution' ] typename = json_result['__typename'] if ( typename == 'StartPipelineExecutionSuccess' or typename == 'LaunchPipelineExecutionSuccess' ): status = DauphinScheduleAttemptStatus.SUCCESS run_id = json_result['run']['runId'] run = graphene_info.schema.type_named('PipelineRun')( graphene_info.context.instance.get_run_by_id(run_id) ) elif typename == 'ScheduleExecutionBlocked': status = DauphinScheduleAttemptStatus.SKIPPED else: status = DauphinScheduleAttemptStatus.ERROR attempts.append( graphene_info.schema.type_named('ScheduleAttempt')( time=os.path.getctime(result_path), json_result=json.dumps(json_result), status=status, run=run, ) ) return attempts def resolve_attempts_count(self, graphene_info): attempt_files = get_schedule_attempt_filenames(graphene_info, self._schedule.name) return len(attempt_files) def resolve_logs_path(self, graphene_info): instance = graphene_info.context.instance repository = graphene_info.context.get_repository() return instance.log_path_for_schedule(repository, self._schedule.name) def resolve_runs(self, graphene_info, **kwargs): return [ graphene_info.schema.type_named('PipelineRun')(r) for r in graphene_info.context.instance.get_runs( filters=PipelineRunsFilter(tags={'dagster/schedule_name': self._schedule.name}), limit=kwargs.get('limit'), ) ] def resolve_runs_count(self, graphene_info): return graphene_info.context.instance.get_runs_count( filter=PipelineRunsFilter(tags=[("dagster/schedule_name", self._schedule.name)]) )
class DauphinPipelineRunLogsSubscriptionFailure(dauphin.ObjectType): class Meta(object): name = 'PipelineRunLogsSubscriptionFailure' message = dauphin.NonNull(dauphin.String) missingRunId = dauphin.Field(dauphin.String)
class DauphinIPipelineSnapshotMixin(object): # Mixin this class to implement IPipelineSnapshot # # Graphene has some strange properties that make it so that you cannot # implement ABCs nor use properties in an overridable way. So the way # the mixin works is that the target classes have to have a method # get_pipeline_index() # def get_pipeline_index(self): raise NotImplementedError() name = dauphin.NonNull(dauphin.String) description = dauphin.String() pipeline_snapshot_id = dauphin.NonNull(dauphin.String) runtime_types = dauphin.non_null_list('RuntimeType') runtime_type_or_error = dauphin.Field( dauphin.NonNull('RuntimeTypeOrError'), runtimeTypeName=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) solids = dauphin.non_null_list('Solid') modes = dauphin.non_null_list('Mode') solid_handles = dauphin.Field( dauphin.non_null_list('SolidHandle'), parentHandleID=dauphin.String() ) solid_handle = dauphin.Field( 'SolidHandle', handleID=dauphin.Argument(dauphin.NonNull(dauphin.String)), ) tags = dauphin.non_null_list('PipelineTag') def resolve_pipeline_snapshot_id(self, _): return self.get_pipeline_index().pipeline_snapshot_id def resolve_name(self, _): return self.get_pipeline_index().name def resolve_description(self, _): return self.get_pipeline_index().description def resolve_runtime_types(self, _graphene_info): # TODO yuhan rename runtime_type in schema pipeline_index = self.get_pipeline_index() return sorted( list( map( lambda dt: to_dauphin_dagster_type(pipeline_index.pipeline_snapshot, dt.key), [t for t in pipeline_index.get_dagster_type_snaps() if t.name], ) ), key=lambda dagster_type: dagster_type.name, ) @capture_dauphin_error def resolve_runtime_type_or_error(self, _, **kwargs): type_name = kwargs['runtimeTypeName'] pipeline_index = self.get_pipeline_index() if not pipeline_index.has_dagster_type_name(type_name): from .errors import DauphinRuntimeTypeNotFoundError raise UserFacingGraphQLError( DauphinRuntimeTypeNotFoundError(runtime_type_name=type_name) ) return to_dauphin_dagster_type( pipeline_index.pipeline_snapshot, pipeline_index.get_dagster_type_from_name(type_name).key, ) def resolve_solids(self, _graphene_info): pipeline_index = self.get_pipeline_index() return build_dauphin_solids(pipeline_index, pipeline_index.dep_structure_index) def resolve_modes(self, _): pipeline_snapshot = self.get_pipeline_index().pipeline_snapshot return [ DauphinMode(pipeline_snapshot.config_schema_snapshot, mode_def_snap) for mode_def_snap in sorted( pipeline_snapshot.mode_def_snaps, key=lambda item: item.name ) ] def resolve_solid_handle(self, _graphene_info, handleID): return _get_solid_handles(self.get_pipeline_index()).get(handleID) def resolve_solid_handles(self, _graphene_info, **kwargs): handles = _get_solid_handles(self.get_pipeline_index()) parentHandleID = kwargs.get('parentHandleID') if parentHandleID == "": handles = {key: handle for key, handle in handles.items() if not handle.parent} elif parentHandleID is not None: handles = { key: handle for key, handle in handles.items() if handle.parent and handle.parent.handleID.to_string() == parentHandleID } return [handles[key] for key in sorted(handles)] def resolve_tags(self, graphene_info): return [ graphene_info.schema.type_named('PipelineTag')(key=key, value=value) for key, value in self.get_pipeline_index().pipeline_snapshot.tags.items() ]
class DauphinPipelineRun(dauphin.ObjectType): class Meta(object): name = 'PipelineRun' runId = dauphin.NonNull(dauphin.String) # Nullable because of historical runs pipelineSnapshotId = dauphin.String() status = dauphin.NonNull('PipelineRunStatus') pipeline = dauphin.NonNull('PipelineReference') stats = dauphin.NonNull('PipelineRunStatsOrError') stepStats = dauphin.non_null_list('PipelineRunStepStats') computeLogs = dauphin.Field( dauphin.NonNull('ComputeLogs'), stepKey=dauphin.Argument(dauphin.NonNull(dauphin.String)), description=''' Compute logs are the stdout/stderr logs for a given solid step computation ''', ) executionPlan = dauphin.Field('ExecutionPlan') stepKeysToExecute = dauphin.List(dauphin.NonNull(dauphin.String)) environmentConfigYaml = dauphin.NonNull(dauphin.String) mode = dauphin.NonNull(dauphin.String) tags = dauphin.non_null_list('PipelineTag') rootRunId = dauphin.Field(dauphin.String) parentRunId = dauphin.Field(dauphin.String) canCancel = dauphin.NonNull(dauphin.Boolean) executionSelection = dauphin.NonNull('ExecutionSelection') def __init__(self, pipeline_run): super(DauphinPipelineRun, self).__init__(runId=pipeline_run.run_id, status=pipeline_run.status, mode=pipeline_run.mode) self._pipeline_run = check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) def resolve_pipeline(self, graphene_info): return get_pipeline_reference_or_raise(graphene_info, self._pipeline_run.selector) def resolve_pipelineSnapshotId(self, _): return self._pipeline_run.pipeline_snapshot_id def resolve_stats(self, graphene_info): return get_stats(graphene_info, self.run_id) def resolve_stepStats(self, graphene_info): return get_step_stats(graphene_info, self.run_id) def resolve_computeLogs(self, graphene_info, stepKey): return graphene_info.schema.type_named('ComputeLogs')( runId=self.run_id, stepKey=stepKey) def resolve_executionPlan(self, graphene_info): if not (self._pipeline_run.execution_plan_snapshot_id and self._pipeline_run.pipeline_snapshot_id): return None from .execution import DauphinExecutionPlan instance = graphene_info.context.instance historical_pipeline = instance.get_historical_pipeline( self._pipeline_run.pipeline_snapshot_id) execution_plan_snapshot = instance.get_execution_plan_snapshot( self._pipeline_run.execution_plan_snapshot_id) return (DauphinExecutionPlan( ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot, represented_pipeline=historical_pipeline, )) if execution_plan_snapshot and historical_pipeline else None) def resolve_stepKeysToExecute(self, _): return self._pipeline_run.step_keys_to_execute def resolve_environmentConfigYaml(self, _graphene_info): return yaml.dump(self._pipeline_run.environment_dict, default_flow_style=False) def resolve_tags(self, graphene_info): return [ graphene_info.schema.type_named('PipelineTag')(key=key, value=value) for key, value in self._pipeline_run.tags.items() ] def resolve_rootRunId(self, _): return self._pipeline_run.root_run_id def resolve_parentRunId(self, _): return self._pipeline_run.parent_run_id @property def run_id(self): return self.runId def resolve_canCancel(self, graphene_info): return graphene_info.context.legacy_environment.execution_manager.can_terminate( self.run_id) def resolve_executionSelection(self, graphene_info): return graphene_info.schema.type_named('ExecutionSelection')( self._pipeline_run.selector)
class DauphinRunningSchedule(dauphin.ObjectType): class Meta(object): name = 'RunningSchedule' schedule_definition = dauphin.NonNull('ScheduleDefinition') python_path = dauphin.Field(dauphin.String) repository_path = dauphin.Field(dauphin.String) status = dauphin.NonNull('ScheduleStatus') runs = dauphin.Field(dauphin.non_null_list('PipelineRun'), limit=dauphin.Int()) runs_count = dauphin.NonNull(dauphin.Int) ticks = dauphin.Field(dauphin.non_null_list('ScheduleTick'), limit=dauphin.Int()) ticks_count = dauphin.NonNull(dauphin.Int) stats = dauphin.NonNull('ScheduleTickStatsSnapshot') # TODO: Delete attempts and attempts_count in 0.8.0 release # https://github.com/dagster-io/dagster/issues/2288 attempts = dauphin.Field(dauphin.non_null_list('ScheduleAttempt'), limit=dauphin.Int()) attempts_count = dauphin.NonNull(dauphin.Int) logs_path = dauphin.NonNull(dauphin.String) def __init__(self, graphene_info, schedule): self._schedule = check.inst_param(schedule, 'schedule', Schedule) super(DauphinRunningSchedule, self).__init__( schedule_definition=graphene_info.schema.type_named( 'ScheduleDefinition')( graphene_info=graphene_info, schedule_def=get_dagster_schedule_def( graphene_info, schedule.name), ), status=schedule.status, python_path=schedule.python_path, repository_path=schedule.repository_path, ) # TODO: Delete in 0.8.0 release # https://github.com/dagster-io/dagster/issues/2288 def resolve_attempts(self, graphene_info, **kwargs): limit = kwargs.get('limit') results = get_schedule_attempt_filenames(graphene_info, self._schedule.name) if limit is None: limit = len(results) latest_results = heapq.nlargest(limit, results, key=os.path.getctime) attempts = [] for result_path in latest_results: with open(result_path, 'r') as f: line = f.readline() if not line: continue # File is empty start_scheduled_execution_response = json.loads(line) run = None if 'errors' in start_scheduled_execution_response: status = DauphinScheduleAttemptStatus.ERROR json_result = start_scheduled_execution_response['errors'] else: json_result = start_scheduled_execution_response['data'][ 'startScheduledExecution'] typename = json_result['__typename'] if (typename == 'StartPipelineRunSuccess' or typename == 'LaunchPipelineRunSuccess'): status = DauphinScheduleAttemptStatus.SUCCESS run_id = json_result['run']['runId'] if graphene_info.context.instance.has_run(run_id): run = graphene_info.schema.type_named( 'PipelineRun')(graphene_info.context.instance. get_run_by_id(run_id)) elif typename == 'ScheduledExecutionBlocked': status = DauphinScheduleAttemptStatus.SKIPPED else: status = DauphinScheduleAttemptStatus.ERROR attempts.append( graphene_info.schema.type_named('ScheduleAttempt')( time=os.path.getctime(result_path), json_result=json.dumps(json_result), status=status, run=run, )) return attempts # TODO: Delete in 0.8.0 release # https://github.com/dagster-io/dagster/issues/2288 def resolve_attempts_count(self, graphene_info): attempt_files = get_schedule_attempt_filenames(graphene_info, self._schedule.name) return len(attempt_files) # TODO: Delete in 0.8.0 release # https://github.com/dagster-io/dagster/issues/2288 def resolve_logs_path(self, graphene_info): instance = graphene_info.context.instance external_repository = graphene_info.context.legacy_external_repository return instance.log_path_for_schedule(external_repository.name, self._schedule.name) def resolve_stats(self, graphene_info): external_repository = graphene_info.context.legacy_external_repository stats = graphene_info.context.instance.get_schedule_tick_stats_by_schedule( external_repository.name, self._schedule.name) return graphene_info.schema.type_named('ScheduleTickStatsSnapshot')( stats) def resolve_ticks(self, graphene_info, limit=None): external_repository = graphene_info.context.legacy_external_repository # TODO: Add cursor limit argument to get_schedule_ticks_by_schedule # https://github.com/dagster-io/dagster/issues/2291 ticks = graphene_info.context.instance.get_schedule_ticks_by_schedule( external_repository.name, self._schedule.name) if not limit: tick_subset = ticks else: tick_subset = ticks[:limit] return [ graphene_info.schema.type_named('ScheduleTick')( tick_id=tick.tick_id, status=tick.status, timestamp=tick.timestamp, tick_specific_data=tick_specific_data_from_dagster_tick( graphene_info, tick), ) for tick in tick_subset ] def resolve_ticks_count(self, graphene_info): external_repository = graphene_info.context.legacy_external_repository ticks = graphene_info.context.instance.get_schedule_ticks_by_schedule( external_repository.name, self._schedule.name) return len(ticks) def resolve_runs(self, graphene_info, **kwargs): return [ graphene_info.schema.type_named('PipelineRun')(r) for r in graphene_info.context.instance.get_runs( filters=PipelineRunsFilter.for_schedule(self._schedule), limit=kwargs.get('limit'), ) ] def resolve_runs_count(self, graphene_info): return graphene_info.context.instance.get_runs_count( filters=PipelineRunsFilter.for_schedule(self._schedule))