def test_dagster_telemetry_unset(caplog):
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd:
yaml.dump({}, fd, default_flow_style=False)
DagsterInstance.local_temp(temp_dir)
runner = CliRunner(env={'DAGSTER_HOME': temp_dir})
with pushd(path_to_file('')):
pipeline_name = 'foo_pipeline'
result = runner.invoke(
pipeline_execute_command,
[
'-f',
path_to_file('test_cli_commands.py'),
'-a',
pipeline_name,
],
)
for record in caplog.records:
message = json.loads(record.getMessage())
if message.get('action') == UPDATE_REPO_STATS:
assert message.get('pipeline_name_hash') == hash_name(
'foo')
assert message.get('num_pipelines_in_repo') == str(1)
assert message.get('repo_hash') == hash_name(
EPHEMERAL_NAME)
assert set(message.keys()) == EXPECTED_KEYS
assert len(caplog.records) == 5
assert result.exit_code == 0
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
return
# TODO move airline demo
script_path = script_relative_path(
'../../../dagster_examples_tests/airline_demo_tests/')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
conn_str = get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test')
wait_for_connection(conn_str)
yield conn_str
def test_dagster_telemetry_unset(caplog):
with tempfile.TemporaryDirectory() as temp_dir:
with instance_for_test(temp_dir=temp_dir,
overrides={"telemetry": {
"enabled": True
}}):
runner = CliRunner(env={"DAGSTER_HOME": temp_dir})
with pushd(path_to_file("")):
pipeline_attribute = "foo_pipeline"
pipeline_name = "foo"
result = runner.invoke(
pipeline_execute_command,
[
"-f",
path_to_file("test_cli_commands.py"), "-a",
pipeline_attribute
],
)
for record in caplog.records:
message = json.loads(record.getMessage())
if message.get("action") == UPDATE_REPO_STATS:
assert message.get("pipeline_name_hash") == hash_name(
pipeline_name)
assert message.get("num_pipelines_in_repo") == str(1)
assert message.get("repo_hash") == hash_name(
get_ephemeral_repository_name(pipeline_name))
assert set(message.keys()) == EXPECTED_KEYS
assert len(caplog.records) == 5
assert result.exit_code == 0
def test_dagster_telemetry_upload(env):
logger = logging.getLogger("dagster_telemetry_logger")
for handler in logger.handlers:
logger.removeHandler(handler)
responses.add(responses.POST, DAGSTER_TELEMETRY_URL)
with environ(env):
with instance_for_test():
runner = CliRunner()
with pushd(path_to_file("")):
pipeline_attribute = "foo_pipeline"
runner.invoke(
pipeline_execute_command,
[
"-f",
path_to_file("test_cli_commands.py"), "-a",
pipeline_attribute
],
)
mock_stop_event = mock.MagicMock()
mock_stop_event.is_set.return_value = False
def side_effect(_):
mock_stop_event.is_set.return_value = True
mock_stop_event.wait.side_effect = side_effect
# Needed to avoid file contention issues on windows with the telemetry log file
cleanup_telemetry_logger()
upload_logs(mock_stop_event, raise_errors=True)
assert responses.assert_call_count(DAGSTER_TELEMETRY_URL, 1)
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path(".")
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
["docker-compose", "stop", "test-postgres-db-airline"])
subprocess.check_output(
["docker-compose", "rm", "-f", "test-postgres-db-airline"])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
["docker-compose", "up", "-d", "test-postgres-db-airline"])
wait_for_connection(
get_conn_string(username="******",
password="******",
hostname=pg_hostname,
db_name="test"))
yield
def postgres(pg_hostname): # pylint: disable=redefined-outer-name
if BUILDKITE:
yield
return
script_path = script_relative_path('.')
if not is_postgres_running():
with pushd(script_path):
try:
subprocess.check_output(
['docker-compose', 'stop', 'test-postgres-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'test-postgres-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(
['docker-compose', 'up', '-d', 'test-postgres-db'])
wait_for_connection(
get_conn_string(username='******',
password='******',
hostname=pg_hostname,
db_name='test'))
yield
def test_pipelines_success(file_path, run_config_path):
with pushd(
file_relative_path(__file__,
"../../../docs_snippets/legacy/data_science/")):
with instance_for_test() as instance:
run_config = load_yaml_from_path(
run_config_path) if run_config_path else {}
recon_pipeline = ReconstructablePipeline.for_file(
file_path, "iris_pipeline")
with tempfile.TemporaryDirectory() as temp_dir:
run_config["resources"] = {
"io_manager": {
"config": {
"base_dir": temp_dir
}
}
}
pipeline_result = execute_pipeline(
recon_pipeline,
run_config=run_config,
instance=instance,
solid_selection=["k_means_iris"
], # skip download_file in tests
)
assert pipeline_result.success
def docker_compose_db():
if sys.version_info.major == 3 and sys.version_info.minor == 7 and CIRCLECI:
yield
return
with pushd(script_relative_path('../')):
subprocess.check_output(
['docker-compose', 'up', '-d', 'airline-demo-db'])
yield
with pushd(script_relative_path('../')):
subprocess.check_output(['docker-compose', 'stop', 'airline-demo-db'])
subprocess.check_output(
['docker-compose', 'rm', '-f', 'airline-demo-db'])
return
def test_runpy(filename, _fn_name, _env_yaml, _mode, _preset, _return_code,
exception):
with pushd(path_to_tutorial_file('')):
if exception:
with pytest.raises(exception):
runpy.run_path(filename, run_name='__main__')
else:
runpy.run_path(filename, run_name='__main__')
def test_runpy(dirname, filename, _fn_name, _env_yaml, _mode, _preset, _return_code, exception):
with pushd(path_to_tutorial_file(dirname)):
filepath = path_to_tutorial_file(os.path.join(dirname, filename))
if exception:
with pytest.raises(exception):
runpy.run_path(filepath, run_name="__main__")
else:
runpy.run_path(filepath, run_name="__main__")
def test_dagster_pipeline_execute(filename, fn_name, env_yaml, mode, preset,
_return_code, _exception):
with pushd(path_to_tutorial_file('')):
dagster_pipeline_execute(
['-f', path_to_tutorial_file(filename), '-n', fn_name] +
(['-e', env_yaml] if env_yaml else []) +
(['-d', mode] if mode else []) +
(['-p', preset] if preset else []))
def test_warehouse_resource(postgres):
run_config = {
'solids': {'read_csv': {'inputs': {'csv_path': {'value': 'cereal.csv'}}}},
'resources': {'warehouse': {'config': {'conn_str': postgres}}},
}
with pushd(
script_relative_path('../../../dagster_examples/intro_tutorial/advanced/pipelines/')
):
result = execute_pipeline(pipeline=modes_pipeline, mode='dev', run_config=run_config,)
assert result.success
if not BUILDKITE:
with pushd(
script_relative_path('../../../dagster_examples/intro_tutorial/advanced/pipelines/')
):
result = execute_pipeline(presets_pipeline, preset='dev')
assert result.success
def test_tutorial_intro_tutorial_hello_world():
with pushd(
script_relative_path(
"../../../docs_snippets/intro_tutorial/basics/single_solid_pipeline/"
)):
result = execute_pipeline(hello_cereal_pipeline)
assert result.success
def test_tutorial_intro_tutorial_hello_world():
with pushd(script_relative_path('../../dagster_examples/intro_tutorial/')):
result = execute_pipeline(hello_cereal_pipeline)
assert result.success
assert len(result.solid_result_list) == 1
assert result.result_for_solid('hello_cereal').output_value() == None
return result
def test_runpy(filename, _fn_name, _env_yaml, _mode, _preset, _return_code, exception):
with pushd(path_to_tutorial_file('')):
try:
runpy.run_path(filename, run_name='__main__')
except Exception as exc:
if exception and isinstance(exc, exception):
return
raise
def test_define_scheduler_repo():
repo = scheduler_repository
assert repo.name == 'hello_cereal_repository'
assert repo.has_pipeline('hello_cereal_pipeline')
with pushd(
script_relative_path(
'../../../docs_snippets/intro_tutorial/advanced/scheduling/')):
result = execute_pipeline(repo.get_pipeline('hello_cereal_pipeline'))
assert result.success
def test_tutorial_intro_tutorial_hello_world():
with pushd(
script_relative_path("../../../docs_snippets/intro_tutorial/basics/e01_first_pipeline/")
):
result = execute_pipeline(hello_cereal_pipeline)
assert result.success
assert len(result.solid_result_list) == 1
assert isinstance(result.result_for_solid("hello_cereal").output_value(), list)
return result
def test_define_repo():
repo = hello_cereal_repository
assert repo.name == "hello_cereal_repository"
assert repo.has_job("hello_cereal_job")
with pushd(
script_relative_path(
"../../../docs_snippets/intro_tutorial/advanced/repositories/")
):
result = repo.get_job("hello_cereal_job").execute_in_process()
assert result.success
def test_execute_pipeline():
execute_kwargs = {
'pipeline_name': ['demo_execution_pipeline'],
'repository_yaml': 'pipeline_execution_repository.yaml',
'module_name': None,
'python_file': None,
'fn_name': None,
}
with pushd(script_relative_path('../../dagster_examples/intro_tutorial/')):
execute_execute_command(['pipeline_execution_env.yaml'], cli_args=execute_kwargs)
def test_define_repo():
repo = hello_cereal_repository
assert repo.name == 'hello_cereal_repository'
assert repo.has_pipeline('hello_cereal_pipeline')
with pushd(
script_relative_path(
'../../../dagster_examples/intro_tutorial/advanced/repositories/'
)):
result = execute_pipeline(repo.get_pipeline('hello_cereal_pipeline'))
assert result.success
def test_yaml_example():
with pushd(script_relative_path('.')):
with scaffold(notebook_name='notebooks/CLI_test_YAML') as notebook_path:
EXPECTED_OUTPUT = EXPECTED_OUTPUT_SHELL.format(
import_statement=EXPECTED_IMPORT_STATEMENT,
declaration_statement='dagstermill.register_repository(define_example_repository())',
)
check_notebook_expected_output(
notebook_path + '.ipynb', expected_output=EXPECTED_OUTPUT
)
def docker_compose_db():
if BUILDKITE:
yield
return
with pushd(script_relative_path('../')):
try:
subprocess.check_output(['docker-compose', 'stop', 'airline-demo-db'])
subprocess.check_output(['docker-compose', 'rm', '-f', 'airline-demo-db'])
except Exception: # pylint: disable=broad-except
pass
subprocess.check_output(['docker-compose', 'up', '-d', 'airline-demo-db'])
yield
with pushd(script_relative_path('../')):
subprocess.check_output(['docker-compose', 'stop', 'airline-demo-db'])
subprocess.check_output(['docker-compose', 'rm', '-f', 'airline-demo-db'])
return
def test_module_example():
with pushd(script_relative_path('.')):
with scaffold(
module_name='dagster',
fn_name='function_name',
notebook_name='notebooks/CLI_test_module',
) as notebook_path:
EXPECTED_OUTPUT = EXPECTED_OUTPUT_SHELL.format(
import_statement='from dagster import function_name',
declaration_statement='dm.register_repository(function_name())',
)
check_notebook_expected_output(notebook_path + '.ipynb',
expected_output=EXPECTED_OUTPUT)
def test_dagster_pipeline_execute(dirname, filename, fn_name, env_yaml, mode,
preset, return_code, _exception):
with pushd(path_to_tutorial_file(dirname)):
filepath = path_to_tutorial_file(os.path.join(dirname, filename))
yamlpath = path_to_tutorial_file(os.path.join(
dirname, env_yaml)) if env_yaml else None
dagster_pipeline_execute(
["-f", filepath, "-a", fn_name] +
(["-c", yamlpath] if yamlpath else []) +
(["--mode", mode] if mode else []) +
(["--preset", preset] if preset else []),
return_code,
)
def test_dagit_logs(
server_mock,
caplog,
):
with seven.TemporaryDirectory() as temp_dir:
with environ({'DAGSTER_HOME': temp_dir}):
with open(os.path.join(temp_dir, 'dagster.yaml'), 'w') as fd:
yaml.dump({}, fd, default_flow_style=False)
DagsterInstance.local_temp(temp_dir)
runner = CliRunner(env={'DAGSTER_HOME': temp_dir})
with pushd(path_to_tutorial_file('')):
result = runner.invoke(
ui,
[
'-w',
file_relative_path(__file__,
'telemetry_repository.yaml'),
],
)
assert result.exit_code == 0, str(result.exception)
actions = set()
for record in caplog.records:
message = json.loads(record.getMessage())
actions.add(message.get('action'))
if message.get('action') == UPDATE_REPO_STATS:
assert message.get('pipeline_name_hash') == ''
assert message.get('num_pipelines_in_repo') == str(4)
assert message.get('repo_hash') == hash_name(
'dagster_test_repository')
assert set(message.keys()) == set([
'action',
'client_time',
'elapsed_time',
'event_id',
'instance_id',
'pipeline_name_hash',
'num_pipelines_in_repo',
'repo_hash',
'python_version',
'metadata',
'version',
])
assert actions == set(
[START_DAGIT_WEBSERVER, UPDATE_REPO_STATS])
assert len(caplog.records) == 2
assert server_mock.call_args_list == [mock.call()]
def test_warehouse_resource(postgres):
run_config = {
"solids": {
"read_csv": {
"inputs": {
"csv_path": {
"value": "cereal.csv"
}
}
}
},
"resources": {
"warehouse": {
"config": {
"conn_str": postgres
}
}
},
}
with pushd(
script_relative_path(
"../../../docs_snippets/intro_tutorial/advanced/pipelines/")):
result = execute_pipeline(
pipeline=modes_pipeline,
mode="dev",
run_config=run_config,
)
assert result.success
if not BUILDKITE:
with pushd(
script_relative_path(
"../../../docs_snippets/intro_tutorial/advanced/pipelines/"
)):
result = execute_pipeline(presets_pipeline, preset="dev")
assert result.success
def test_pipelines_success(file_path, run_config_path):
with pushd(
file_relative_path(__file__,
'../../../docs_snippets/legacy/data_science/')):
instance = DagsterInstance.local_temp()
run_config = load_yaml_from_path(
run_config_path) if run_config_path else None
recon_pipeline = ReconstructablePipeline.for_file(
file_path, 'iris_pipeline')
pipeline_result = execute_pipeline(recon_pipeline,
run_config=run_config,
instance=instance)
assert pipeline_result.success
def test_define_scheduler_repo():
repo = scheduler_repository
assert repo.name == "hello_cereal_repository"
assert repo.has_job("hello_cereal_job")
with pushd(
script_relative_path(
"../../../docs_snippets/intro_tutorial/advanced/scheduling/")):
result = repo.get_job("hello_cereal_job", ).execute_in_process(
{"ops": {
"hello_cereal": {
"config": {
"date": "date"
}
}
}})
assert result.success
def test_scaffold():
with pushd(file_relative_path(__file__, ".")):
with scaffold(
notebook_name="notebooks/cli_test_scaffold") as notebook_path:
check_notebook_expected_output(notebook_path + ".ipynb")
with scaffold(notebook_name="notebooks/cli_test_scaffold",
kernel="dagster") as notebook_path:
check_notebook_expected_output(notebook_path + ".ipynb")
with pytest.raises(
CheckError,
match=re.escape(
"Could not find kernel 'foobar': available kernels are")):
with scaffold(notebook_name="notebooks/cli_test_scaffold",
kernel="foobar") as _:
pass
def test_dagster_telemetry_no_test_env_upload(env):
with instance_for_test():
with environ(env):
runner = CliRunner()
with pushd(path_to_file("")):
pipeline_attribute = "foo_pipeline"
runner.invoke(
pipeline_execute_command,
[
"-f",
path_to_file("test_cli_commands.py"), "-a",
pipeline_attribute
],
)
upload_logs(mock.MagicMock())
assert responses.assert_call_count(DAGSTER_TELEMETRY_URL, 0)