def test_roundtrip_run():
run = PipelineRun(
pipeline_name='pipey_mcpipeface',
run_id='8675309',
environment_dict={'good': True},
mode='default',
selector=ExecutionSelector('pipey_mcpipeface'),
step_keys_to_execute=['step_1', 'step_2', 'step_3'],
tags={'tag_it': 'bag_it'},
status=PipelineRunStatus.NOT_STARTED,
previous_run_id='previousID',
)
for field in run:
# ensure we have a test value to round trip for each field
assert field
exec_params = execution_params_from_pipeline_run(run)
assert run == pipeline_run_from_execution_params(exec_params)
exec_params_gql = execution_params_from_graphql(
exec_params.to_graphql_input())
assert exec_params_gql == exec_params
assert run == pipeline_run_from_execution_params(exec_params_gql)
empty_run = PipelineRun.create_empty_run('foo', 'bar')
exec_params = execution_params_from_pipeline_run(empty_run)
assert empty_run == pipeline_run_from_execution_params(exec_params)
exec_params_gql = execution_params_from_graphql(
exec_params.to_graphql_input())
assert exec_params_gql == exec_params
assert empty_run == pipeline_run_from_execution_params(exec_params_gql)
def test_roundtrip_run():
run_with_snapshot = PipelineRun(
pipeline_name='pipey_mcpipeface',
run_id='8675309',
environment_dict={'good': True},
mode='default',
solid_subset=['solid_1'],
step_keys_to_execute=['step_1', 'step_2', 'step_3'],
tags={'tag_it': 'bag_it'},
status=PipelineRunStatus.NOT_STARTED,
root_run_id='previousID',
parent_run_id='previousID',
pipeline_snapshot_id='pipey_mcpipeface_snapshot_id',
execution_plan_snapshot_id='mcexecutionplanface_snapshot_id',
)
for field in run_with_snapshot:
# ensure we have a test value to round trip for each field
assert field
# The invariant that all the execution parameter structs
# pipeline run can be constructed from each other is no longer
# true. Clients of the GraphQL API cannot know the value of the
# pipeline_snapshot_id prior to execution, because it is
# constructed on the server. Hence these roundtrip tests
# do not include snapshot_id
run = run_with_snapshot._replace(pipeline_snapshot_id=None,
execution_plan_snapshot_id=None)
exec_params = execution_params_from_pipeline_run(run)
for key, value in pipeline_run_args_from_execution_params(
exec_params).items():
assert getattr(run, key) == value
exec_params_gql = execution_params_from_graphql(
exec_params.to_graphql_input())
assert exec_params_gql == exec_params
for key, value in pipeline_run_args_from_execution_params(
exec_params_gql).items():
assert getattr(run, key) == value
empty_run = PipelineRun(pipeline_name='foo', run_id='bar', mode='default')
exec_params = execution_params_from_pipeline_run(empty_run)
for key, value in pipeline_run_args_from_execution_params(
exec_params).items():
assert getattr(empty_run, key) == value
exec_params_gql = execution_params_from_graphql(
exec_params.to_graphql_input())
assert exec_params_gql == exec_params
for key, value in pipeline_run_args_from_execution_params(
exec_params_gql).items():
assert getattr(empty_run, key) == value
def launch_run(self, instance, run):
self.validate()
execution_params = execution_params_from_pipeline_run(run)
variables = {'executionParams': execution_params.to_graphql_input()}
instance.create_run(run)
response = requests.post(
urljoin(self._address, '/graphql'),
params={
'query': START_PIPELINE_EXECUTION_MUTATION,
'variables': seven.json.dumps(variables),
},
timeout=self._timeout,
)
response.raise_for_status()
result = response.json()['data']['startPipelineExecution']
if result['__typename'] == 'StartPipelineExecutionSuccess':
return run.run_with_status(
PipelineRunStatus(result['run']['status']))
raise DagsterLaunchFailedError(
'Failed to launch run with {cls} targeting {address}:\n{result}'.
format(cls=self.__class__.__name__,
address=self._address,
result=result))
def test_roundtrip_run():
run = PipelineRun(
pipeline_name='pipey_mcpipeface',
run_id='8675309',
environment_dict={'good': True},
mode='default',
selector=ExecutionSelector('pipey_mcpipeface'),
step_keys_to_execute=['step_1', 'step_2', 'step_3'],
tags={'tag_it': 'bag_it'},
status=PipelineRunStatus.NOT_STARTED,
)
assert run == pipeline_run_from_execution_params(
execution_params_from_pipeline_run(run))
empty_run = PipelineRun.create_empty_run('foo', 'bar')
assert empty_run == pipeline_run_from_execution_params(
execution_params_from_pipeline_run(empty_run))
def _start_pipeline_execution_for_created_run(graphene_info, run_id):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
instance = graphene_info.context.instance
execution_manager_settings = instance.dagit_settings.get(
'execution_manager')
if execution_manager_settings and execution_manager_settings.get(
'disabled'):
return graphene_info.schema.type_named(
'StartPipelineExecutionDisabledError')()
run = instance.get_run_by_id(run_id)
if not run:
return graphene_info.schema.type_named('PipelineRunNotFoundError')(
run_id)
pipeline_def = get_pipeline_def_from_selector(graphene_info, run.selector)
get_validated_config(
graphene_info,
pipeline_def,
environment_dict=run.environment_dict,
mode=run.mode,
)
execution_plan = create_execution_plan(
pipeline_def,
run.environment_dict,
run_config=RunConfig(
mode=run.mode,
previous_run_id=run.previous_run_id,
tags=run.tags,
),
)
execution_params = execution_params_from_pipeline_run(run)
_check_start_pipeline_execution_errors(graphene_info, execution_params,
execution_plan)
graphene_info.context.execution_manager.execute_pipeline(
graphene_info.context.get_handle(),
pipeline_def,
run,
instance=instance,
)
return graphene_info.schema.type_named('StartPipelineExecutionSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(run))
def construct_job(self, run):
check.inst_param(run, 'run', PipelineRun)
dagster_labels = {
'app.kubernetes.io/name': 'dagster',
'app.kubernetes.io/instance': 'dagster',
'app.kubernetes.io/version': dagster_version,
}
execution_params = execution_params_from_pipeline_run(run)
job_container = client.V1Container(
name='dagster-job-%s' % run.run_id,
image=self.job_image,
command=['dagster-graphql'],
args=[
"-p",
"startPipelineExecution",
"-v",
json.dumps(
{'executionParams': execution_params.to_graphql_input()}),
],
image_pull_policy=self.image_pull_policy,
env=[
client.V1EnvVar(
name='DAGSTER_PG_PASSWORD',
value_from=client.V1EnvVarSource(
secret_key_ref=client.V1SecretKeySelector(
name='dagster-postgresql',
key='postgresql-password')),
),
],
env_from=self.env_from_sources,
volume_mounts=[
client.V1VolumeMount(
name='dagster-instance',
mount_path='{dagster_home}/dagster.yaml'.format(
dagster_home=self.dagster_home),
sub_path='dagster.yaml',
)
],
)
config_map_volume = client.V1Volume(
name='dagster-instance',
config_map=client.V1ConfigMapVolumeSource(
name=self.instance_config_map),
)
template = client.V1PodTemplateSpec(
metadata=client.V1ObjectMeta(
name='dagster-job-pod-%s' % run.run_id,
labels=dagster_labels,
),
spec=client.V1PodSpec(
image_pull_secrets=self.image_pull_secrets,
service_account_name=self.service_account_name,
restart_policy='Never',
containers=[job_container],
volumes=[config_map_volume],
),
)
job = client.V1Job(
api_version="batch/v1",
kind="Job",
metadata=client.V1ObjectMeta(name='dagster-job-%s' % run.run_id,
labels=dagster_labels),
spec=client.V1JobSpec(
template=template,
backoff_limit=BACKOFF_LIMIT,
ttl_seconds_after_finished=TTL_SECONDS_AFTER_FINISHED,
),
)
return job
