def test_fn_is_wrong_thing():
with pytest.raises(
DagsterInvariantViolationError,
match='must resolve to a PipelineDefinition',
):
execute_execute_command(
env=[],
cli_args={
'repository_yaml': None,
'pipeline_name': (),
'python_file': file_relative_path(__file__,
'test_cli_commands.py'),
'module_name': None,
'fn_name': 'not_a_repo_or_pipeline',
},
)
def test_attribute_not_found():
with instance_for_test() as instance:
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape("nope not found at module scope in file"),
):
execute_execute_command(
kwargs={
"repository_yaml": None,
"pipeline": None,
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"module_name": None,
"attribute": "nope",
},
instance=instance,
)
def test_more_than_one_pipeline():
with instance_for_test() as instance:
with pytest.raises(
UsageError,
match=re.escape("Must provide --pipeline as there is more than one pipeline in bar. "),
):
execute_execute_command(
kwargs={
"repository_yaml": None,
"pipeline": None,
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"module_name": None,
"attribute": None,
},
instance=instance,
)
def test_fn_returns_wrong_thing():
with pytest.raises(DagsterInvariantViolationError) as exc_info:
execute_execute_command(
env={},
cli_args={
'repository_yaml': None,
'pipeline_name': (),
'python_file': script_relative_path('test_cli_commands.py'),
'module_name': None,
'fn_name': 'not_a_repo_or_pipeline_fn',
},
)
assert str(exc_info.value) == (
'not_a_repo_or_pipeline_fn is a function but must return a '
'PipelineDefinition or a RepositoryDefinition, or be decorated '
'with @pipeline.')
def test_attribute_fn_returns_wrong_thing():
with pytest.raises(
DagsterIPCProtocolError,
match=re.escape(
"Loadable attributes must be either a PipelineDefinition or a RepositoryDefinition."
),
):
execute_execute_command(
env=[],
cli_args={
'repository_yaml': None,
'pipeline': None,
'python_file': file_relative_path(__file__, 'test_cli_commands.py'),
'module_name': None,
'attribute': 'not_a_repo_or_pipeline_fn',
},
)
def test_attribute_fn_returns_wrong_thing():
with instance_for_test() as instance:
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
"Loadable attributes must be either a PipelineDefinition or a RepositoryDefinition."
),
):
execute_execute_command(
kwargs={
"repository_yaml": None,
"pipeline": None,
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"module_name": None,
"attribute": "not_a_repo_or_pipeline_fn",
},
instance=instance,
)
def test_attribute_is_wrong_thing():
with pytest.raises(
DagsterInvariantViolationError,
match=re.escape(
'Loadable attributes must be either a PipelineDefinition or a '
'RepositoryDefinition. Got 123.'
),
):
execute_execute_command(
env=[],
cli_args={
'repository_yaml': None,
'pipeline': None,
'python_file': file_relative_path(__file__, 'test_cli_commands.py'),
'module_name': None,
'attribute': 'not_a_repo_or_pipeline',
},
)
def test_fn_is_wrong_thing():
with pytest.raises(DagsterInvariantViolationError) as exc_info:
execute_execute_command(
env={'execution': {'in_process': {'config': {'raise_on_error': True}}}},
cli_args={
'repository_yaml': None,
'pipeline_name': (),
'python_file': script_relative_path('test_cli_commands.py'),
'module_name': None,
'fn_name': 'not_a_repo_or_pipeline',
},
)
assert str(exc_info.value) == (
'not_a_repo_or_pipeline must be a function that returns a '
'PipelineDefinition or a RepositoryDefinition, or a function '
'decorated with @pipeline.'
)
def test_execute_command():
for cli_args in valid_execute_args():
execute_execute_command(env=None, cli_args=cli_args)
for cli_args in valid_execute_args():
execute_execute_command(
env=[file_relative_path(__file__, 'default_log_error_env.yaml')], cli_args=cli_args
)
runner = CliRunner()
for cli_args in valid_cli_args():
runner_pipeline_execute(runner, cli_args)
runner_pipeline_execute(
runner,
['--config', file_relative_path(__file__, 'default_log_error_env.yaml')] + cli_args,
)
def test_more_than_one_pipeline():
with pytest.raises(
UsageError,
match=re.escape(
"Must provide --pipeline as there is more than one pipeline in bar. "
"Options are: ['baz', 'foo']."
),
):
execute_execute_command(
env=None,
cli_args={
'repository_yaml': None,
'pipeline': None,
'python_file': file_relative_path(__file__, 'test_cli_commands.py'),
'module_name': None,
'attribute': None,
},
)
def test_execute_command():
for cli_args, uses_legacy_repository_yaml_format in valid_execute_args():
if uses_legacy_repository_yaml_format:
with pytest.warns(
UserWarning,
match=re.escape(
'You are using the legacy repository yaml format. Please update your file '
),
):
execute_execute_command(env=None, cli_args=cli_args)
else:
execute_execute_command(env=None, cli_args=cli_args)
for cli_args, uses_legacy_repository_yaml_format in valid_execute_args():
if uses_legacy_repository_yaml_format:
with pytest.warns(
UserWarning,
match=re.escape(
'You are using the legacy repository yaml format. Please update your file '
),
):
execute_execute_command(
env=[file_relative_path(__file__, 'default_log_error_env.yaml')],
cli_args=cli_args,
)
else:
execute_execute_command(
env=[file_relative_path(__file__, 'default_log_error_env.yaml')], cli_args=cli_args
)
runner = CliRunner()
for cli_args, uses_legacy_repository_yaml_format in valid_cli_args():
if uses_legacy_repository_yaml_format:
with pytest.warns(
UserWarning,
match=re.escape(
'You are using the legacy repository yaml format. Please update your file '
),
):
runner_pipeline_execute(runner, cli_args)
runner_pipeline_execute(
runner,
['--config', file_relative_path(__file__, 'default_log_error_env.yaml')]
+ cli_args,
)
else:
runner_pipeline_execute(runner, cli_args)
runner_pipeline_execute(
runner,
['--config', file_relative_path(__file__, 'default_log_error_env.yaml')] + cli_args,
)
def test_default_memory_run_storage():
with instance_for_test() as instance:
cli_args = {
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"atribute": "bar",
"pipeline": "foo",
"module_name": None,
}
result = execute_execute_command(kwargs=cli_args, instance=instance)
assert result.success
def test_output_execute_log_stderr(capfd):
with instance_for_test(
overrides={
"compute_logs": {
"module": "dagster.core.storage.noop_compute_log_manager",
"class": "NoOpComputeLogManager",
}
},
) as instance:
with pytest.raises(click.ClickException, match=re.escape("resulted in failure")):
execute_execute_command(
kwargs={
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"attribute": "stderr_pipeline",
},
instance=instance,
)
captured = capfd.readouterr()
assert "I AM SUPPOSED TO FAIL" in captured.err
def test_output_execute_log_stdout(capfd):
with instance_for_test(overrides={
"compute_logs": {
"module": "dagster.core.storage.noop_compute_log_manager",
"class": "NoOpComputeLogManager",
}
}, ) as instance:
execute_execute_command(
kwargs={
"python_file": file_relative_path(__file__,
"test_cli_commands.py"),
"attribute": "stdout_pipeline",
},
instance=instance,
)
captured = capfd.readouterr()
# All pipeline execute output currently logged to stderr
assert "HELLO WORLD" in captured.err
def test_execute_command():
for cli_args in valid_execute_args():
execute_execute_command(env=None, raise_on_error=True, cli_args=cli_args)
for cli_args in valid_execute_args():
execute_execute_command(
env=[script_relative_path('default_log_error_env.yaml')],
raise_on_error=True,
cli_args=cli_args,
)
runner = CliRunner()
for cli_args in valid_cli_args():
runner_pipeline_execute(runner, cli_args)
runner_pipeline_execute(
runner, ['--env', script_relative_path('default_log_error_env.yaml')] + cli_args
)
def test_default_memory_run_storage():
cli_args = {
'repository_yaml': script_relative_path('repository_file.yaml'),
'pipeline_name': ('foo', ),
'python_file': None,
'module_name': None,
'fn_name': None,
}
result = execute_execute_command(env=None, cli_args=cli_args)
assert result.success
def test_default_memory_run_storage():
cli_args = {
'workspace': file_relative_path(__file__, 'repository_file.yaml'),
'pipeline': 'foo',
'python_file': None,
'module_name': None,
'attribute': None,
}
result = execute_execute_command(env=None, cli_args=cli_args)
assert result.success
def test_override_with_filesystem_storage():
cli_args = {
'repository_yaml': script_relative_path('repository_file.yaml'),
'pipeline_name': ('foo', ),
'python_file': None,
'module_name': None,
'fn_name': None,
}
result = execute_execute_command(
env=[script_relative_path('filesystem_env.yaml')], cli_args=cli_args)
assert result.success
def test_override_with_filesystem_storage():
with instance_for_test() as instance:
cli_args = {
"python_file": file_relative_path(__file__, "test_cli_commands.py"),
"atribute": "bar",
"pipeline": "foo",
"module_name": None,
"config": (file_relative_path(__file__, "filesystem_env.yaml"),),
}
result = execute_execute_command(kwargs=cli_args, instance=instance,)
assert result.success
def test_execute_command():
for cli_args in valid_execute_args():
execute_execute_command(env=None,
raise_on_error=True,
cli_args=cli_args)
for cli_args in valid_execute_args():
execute_execute_command(env=[script_relative_path('env.yml')],
raise_on_error=True,
cli_args=cli_args)
runner = CliRunner()
for cli_args in valid_cli_args():
result = runner.invoke(pipeline_execute_command, cli_args)
assert result.exit_code == 0
result = runner.invoke(
pipeline_execute_command,
['--env', script_relative_path('env.yml')] + cli_args)
assert result.exit_code == 0
def test_override_with_filesystem_storage():
cli_args = {
'workspace': file_relative_path(__file__, 'repository_file.yaml'),
'pipeline': 'foo',
'python_file': None,
'module_name': None,
'attribute': None,
}
result = execute_execute_command(
env=[file_relative_path(__file__, 'filesystem_env.yaml')], cli_args=cli_args
)
assert result.success
def test_override_with_in_memory_storage():
cli_args = {
'repository_yaml': file_relative_path(__file__, 'repository_file.yaml'),
'pipeline_name': ('foo',),
'python_file': None,
'module_name': None,
'fn_name': None,
}
result = execute_execute_command(
env=[file_relative_path(__file__, 'in_memory_env.yaml')], cli_args=cli_args
)
assert result.success
def test_job_command_only_selects_job():
with instance_for_test() as instance:
job_kwargs = {
"workspace": None,
"pipeline_or_job": "my_job",
"python_file": file_relative_path(__file__,
"repo_pipeline_and_job.py"),
"module_name": None,
"attribute": "my_repo",
}
pipeline_kwargs = job_kwargs.copy()
pipeline_kwargs["pipeline_or_job"] = "my_pipeline"
result = execute_execute_command(kwargs=job_kwargs,
instance=instance,
using_job_op_graph_apis=True)
assert result.success
with pytest.raises(Exception, match="not found in repository"):
execute_execute_command(kwargs=pipeline_kwargs,
instance=instance,
using_job_op_graph_apis=True)
def test_default_memory_run_storage():
cli_args = {
'repository_yaml': script_relative_path('repository_file.yaml'),
'pipeline_name': ('foo',),
'python_file': None,
'module_name': None,
'fn_name': None,
}
result = execute_execute_command(env=None, raise_on_error=True, cli_args=cli_args)
assert result.success
run_dir = os.path.join(base_runs_directory(), result.run_id)
assert not os.path.isdir(run_dir)
def test_override_with_filesystem_storage():
cli_args = {
'repository_yaml': script_relative_path('repository_file.yaml'),
'pipeline_name': ('foo', ),
'python_file': None,
'module_name': None,
'fn_name': None,
}
result = execute_execute_command(
env=[script_relative_path('filesystem_env.yaml')], cli_args=cli_args)
assert result.success
run_dir = os.path.join(base_runs_directory(), result.run_id)
assert os.path.exists(run_dir)
def test_default_memory_run_storage():
cli_args = {
'workspace': (file_relative_path(__file__, 'repository_file.yaml'), ),
'pipeline': 'foo',
'python_file': None,
'module_name': None,
'attribute': None,
}
with pytest.warns(
UserWarning,
match=re.escape(
'You are using the legacy repository yaml format. Please update your file '
),
):
result = execute_execute_command(env=None, cli_args=cli_args)
assert result.success
def test_execute_command_no_env(gen_execute_args):
with gen_execute_args as (cli_args, instance):
execute_execute_command(kwargs=cli_args, instance=instance)
def job_execute_command(**kwargs):
with capture_interrupts():
with get_instance_for_service("``dagster job execute``") as instance:
execute_execute_command(instance, kwargs, True)
def test_job_execute_command_no_env(gen_execute_args):
with gen_execute_args as (cli_args, instance):
execute_execute_command(kwargs=cli_args,
instance=instance,
using_job_op_graph_apis=True)
def test_execute_non_existant_file():
with instance_for_test() as instance:
kwargs = non_existant_python_origin_target_args()
with pytest.raises(OSError):
execute_execute_command(kwargs=kwargs, instance=instance)
