def test_roundtrip_run(): run = PipelineRun( pipeline_name='pipey_mcpipeface', run_id='8675309', environment_dict={'good': True}, mode='default', selector=ExecutionSelector('pipey_mcpipeface'), step_keys_to_execute=['step_1', 'step_2', 'step_3'], tags={'tag_it': 'bag_it'}, status=PipelineRunStatus.NOT_STARTED, previous_run_id='previousID', ) for field in run: # ensure we have a test value to round trip for each field assert field exec_params = execution_params_from_pipeline_run(run) assert run == pipeline_run_from_execution_params(exec_params) exec_params_gql = execution_params_from_graphql( exec_params.to_graphql_input()) assert exec_params_gql == exec_params assert run == pipeline_run_from_execution_params(exec_params_gql) empty_run = PipelineRun.create_empty_run('foo', 'bar') exec_params = execution_params_from_pipeline_run(empty_run) assert empty_run == pipeline_run_from_execution_params(exec_params) exec_params_gql = execution_params_from_graphql( exec_params.to_graphql_input()) assert exec_params_gql == exec_params assert empty_run == pipeline_run_from_execution_params(exec_params_gql)
def test_roundtrip_run(): run_with_snapshot = PipelineRun( pipeline_name='pipey_mcpipeface', run_id='8675309', environment_dict={'good': True}, mode='default', solid_subset=['solid_1'], step_keys_to_execute=['step_1', 'step_2', 'step_3'], tags={'tag_it': 'bag_it'}, status=PipelineRunStatus.NOT_STARTED, root_run_id='previousID', parent_run_id='previousID', pipeline_snapshot_id='pipey_mcpipeface_snapshot_id', execution_plan_snapshot_id='mcexecutionplanface_snapshot_id', ) for field in run_with_snapshot: # ensure we have a test value to round trip for each field assert field # The invariant that all the execution parameter structs # pipeline run can be constructed from each other is no longer # true. Clients of the GraphQL API cannot know the value of the # pipeline_snapshot_id prior to execution, because it is # constructed on the server. Hence these roundtrip tests # do not include snapshot_id run = run_with_snapshot._replace(pipeline_snapshot_id=None, execution_plan_snapshot_id=None) exec_params = execution_params_from_pipeline_run(run) for key, value in pipeline_run_args_from_execution_params( exec_params).items(): assert getattr(run, key) == value exec_params_gql = execution_params_from_graphql( exec_params.to_graphql_input()) assert exec_params_gql == exec_params for key, value in pipeline_run_args_from_execution_params( exec_params_gql).items(): assert getattr(run, key) == value empty_run = PipelineRun(pipeline_name='foo', run_id='bar', mode='default') exec_params = execution_params_from_pipeline_run(empty_run) for key, value in pipeline_run_args_from_execution_params( exec_params).items(): assert getattr(empty_run, key) == value exec_params_gql = execution_params_from_graphql( exec_params.to_graphql_input()) assert exec_params_gql == exec_params for key, value in pipeline_run_args_from_execution_params( exec_params_gql).items(): assert getattr(empty_run, key) == value
def launch_run(self, instance, run): self.validate() execution_params = execution_params_from_pipeline_run(run) variables = {'executionParams': execution_params.to_graphql_input()} instance.create_run(run) response = requests.post( urljoin(self._address, '/graphql'), params={ 'query': START_PIPELINE_EXECUTION_MUTATION, 'variables': seven.json.dumps(variables), }, timeout=self._timeout, ) response.raise_for_status() result = response.json()['data']['startPipelineExecution'] if result['__typename'] == 'StartPipelineExecutionSuccess': return run.run_with_status( PipelineRunStatus(result['run']['status'])) raise DagsterLaunchFailedError( 'Failed to launch run with {cls} targeting {address}:\n{result}'. format(cls=self.__class__.__name__, address=self._address, result=result))
def test_roundtrip_run(): run = PipelineRun( pipeline_name='pipey_mcpipeface', run_id='8675309', environment_dict={'good': True}, mode='default', selector=ExecutionSelector('pipey_mcpipeface'), step_keys_to_execute=['step_1', 'step_2', 'step_3'], tags={'tag_it': 'bag_it'}, status=PipelineRunStatus.NOT_STARTED, ) assert run == pipeline_run_from_execution_params( execution_params_from_pipeline_run(run)) empty_run = PipelineRun.create_empty_run('foo', 'bar') assert empty_run == pipeline_run_from_execution_params( execution_params_from_pipeline_run(empty_run))
def _start_pipeline_execution_for_created_run(graphene_info, run_id): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) instance = graphene_info.context.instance execution_manager_settings = instance.dagit_settings.get( 'execution_manager') if execution_manager_settings and execution_manager_settings.get( 'disabled'): return graphene_info.schema.type_named( 'StartPipelineExecutionDisabledError')() run = instance.get_run_by_id(run_id) if not run: return graphene_info.schema.type_named('PipelineRunNotFoundError')( run_id) pipeline_def = get_pipeline_def_from_selector(graphene_info, run.selector) get_validated_config( graphene_info, pipeline_def, environment_dict=run.environment_dict, mode=run.mode, ) execution_plan = create_execution_plan( pipeline_def, run.environment_dict, run_config=RunConfig( mode=run.mode, previous_run_id=run.previous_run_id, tags=run.tags, ), ) execution_params = execution_params_from_pipeline_run(run) _check_start_pipeline_execution_errors(graphene_info, execution_params, execution_plan) graphene_info.context.execution_manager.execute_pipeline( graphene_info.context.get_handle(), pipeline_def, run, instance=instance, ) return graphene_info.schema.type_named('StartPipelineExecutionSuccess')( run=graphene_info.schema.type_named('PipelineRun')(run))
def construct_job(self, run): check.inst_param(run, 'run', PipelineRun) dagster_labels = { 'app.kubernetes.io/name': 'dagster', 'app.kubernetes.io/instance': 'dagster', 'app.kubernetes.io/version': dagster_version, } execution_params = execution_params_from_pipeline_run(run) job_container = client.V1Container( name='dagster-job-%s' % run.run_id, image=self.job_image, command=['dagster-graphql'], args=[ "-p", "startPipelineExecution", "-v", json.dumps( {'executionParams': execution_params.to_graphql_input()}), ], image_pull_policy=self.image_pull_policy, env=[ client.V1EnvVar( name='DAGSTER_PG_PASSWORD', value_from=client.V1EnvVarSource( secret_key_ref=client.V1SecretKeySelector( name='dagster-postgresql', key='postgresql-password')), ), ], env_from=self.env_from_sources, volume_mounts=[ client.V1VolumeMount( name='dagster-instance', mount_path='{dagster_home}/dagster.yaml'.format( dagster_home=self.dagster_home), sub_path='dagster.yaml', ) ], ) config_map_volume = client.V1Volume( name='dagster-instance', config_map=client.V1ConfigMapVolumeSource( name=self.instance_config_map), ) template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( name='dagster-job-pod-%s' % run.run_id, labels=dagster_labels, ), spec=client.V1PodSpec( image_pull_secrets=self.image_pull_secrets, service_account_name=self.service_account_name, restart_policy='Never', containers=[job_container], volumes=[config_map_volume], ), ) job = client.V1Job( api_version="batch/v1", kind="Job", metadata=client.V1ObjectMeta(name='dagster-job-%s' % run.run_id, labels=dagster_labels), spec=client.V1JobSpec( template=template, backoff_limit=BACKOFF_LIMIT, ttl_seconds_after_finished=TTL_SECONDS_AFTER_FINISHED, ), ) return job