def test_cache_file_from_s3_step_one_two():
boto_s3 = mock.MagicMock()
# mock.patch is difficult to get right and requires monkeypatching of global artifacts
with get_temp_dir() as temp_dir, mock.patch(
file_cache_folder.__module__ + '.file_cache_folder', new=lambda: temp_dir
), mock.patch('boto3.client', new=lambda *_args, **_kwargs: boto_s3):
execute_solid(
cache_file_from_s3,
input_values=dict(s3_coord={'bucket': 'some-bucket', 'key': 'some-key'}),
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_invalid_path_prefix():
with mock.patch("gevent.pywsgi.WSGIServer"), seven.TemporaryDirectory(
) as temp_dir:
instance = DagsterInstance.get(temp_dir)
with load_workspace_from_yaml_paths(
[file_relative_path(__file__, "./workspace.yaml")],
UserProcessApi.CLI,
) as workspace:
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace(
instance=instance,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix="no-leading-slash",
)
assert "path prefix should begin with a leading" in str(
exc_info.value)
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace(
instance=instance,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix="/extra-trailing-slash/",
)
assert "path prefix should not include a trailing" in str(
exc_info.value)
def test_no_additional_warn_text():
with mock.patch('warnings.warn') as warn_mock:
rename_warning('a_new_name', 'an_old_name', '0.3.0')
warn_mock.assert_called_once_with(
'"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" instead.',
stacklevel=3,
)
with mock.patch('warnings.warn') as warn_mock:
rename_warning('a_new_name', 'an_old_name', '0.3.0',
'Additional compelling text.')
warn_mock.assert_called_once_with(
('"an_old_name" is deprecated and will be removed in 0.3.0, use "a_new_name" '
'instead. Additional compelling text.'),
stacklevel=3,
)
def test_backcompat_old_flag():
with mock.patch('warnings.warn') as warn_mock:
assert is_new(old_flag=False) is True
warn_mock.assert_called_once_with(
'"old_flag" is deprecated and will be removed in 0.9.0, use "new_flag" instead. Will remove at next release.',
stacklevel=3,
)
def test_papertrail_logger():
with mock.patch('logging.handlers.SysLogHandler.emit') as emit:
execute_pipeline(
hello_pipeline,
{
'loggers': {
'console': {'config': {'log_level': 'INFO'}},
'papertrail': {
'config': {
'log_level': 'INFO',
'name': 'hello_pipeline',
'papertrail_address': '127.0.0.1',
'papertrail_port': 12345,
}
},
}
},
)
log_record = emit.call_args_list[0][0][0]
assert isinstance(log_record, logging.LogRecord)
assert log_record.name == 'hello_pipeline'
assert log_record.levelname == 'INFO'
for msg in [
'orig_message = "Hello, world!"',
'pipeline = "hello_pipeline"',
'step_key = "hello_logs.compute"',
'solid = "hello_logs"',
'solid_definition = "hello_logs"',
]:
assert msg in log_record.msg
def test_conditional_execution():
with mock.patch("random.randint", return_value=1):
result = execute_pipeline(my_pipeline)
assert (result.events_by_step_key["branch_2_solid.compute"]
[0].event_type_value != "STEP_SKIPPED")
assert (result.events_by_step_key["branch_1_solid.compute"]
[0].event_type_value == "STEP_SKIPPED")
def test_cache_file_from_s3_step_two_use_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
'boto3.client', new=lambda *_args, **_kwargs: boto_s3):
execute_solid(
cache_file_from_s3,
ModeDefinition(resource_defs={'file_cache': fs_file_cache}),
environment_dict={
'resources': {
'file_cache': {
'config': {
'target_folder': temp_dir
}
}
},
'solids': {
'cache_file_from_s3': {
'inputs': {
's3_coord': {
'bucket': 'some-bucket',
'key': 'some-key'
}
}
}
},
},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_invalid_path_prefix():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
workspace = load_workspace_from_yaml_paths(
[file_relative_path(__file__, './workspace.yaml')])
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace(
storage_fallback=temp_dir,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix='no-leading-slash',
)
assert 'path prefix should begin with a leading' in str(exc_info.value)
with pytest.raises(Exception) as exc_info:
host_dagit_ui_with_workspace(
storage_fallback=temp_dir,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix='/extra-trailing-slash/',
)
assert 'path prefix should not include a trailing' in str(
exc_info.value)
def test_papertrail_logger():
with mock.patch('logging.handlers.SysLogHandler.emit') as emit:
execute_pipeline(
hello_pipeline,
{
'loggers': {
'console': {
'config': {
'log_level': 'INFO'
}
},
'papertrail': {
'config': {
'log_level': 'INFO',
'name': 'hello_pipeline',
'papertrail_address': '127.0.0.1',
'papertrail_port': 12345,
}
},
}
},
run_config=RunConfig(run_id='123'),
)
log_record = emit.call_args_list[0][0][0]
assert isinstance(log_record, logging.LogRecord)
assert log_record.name == 'hello_pipeline'
assert log_record.levelname == 'INFO'
assert (log_record.msg == '''system - 123 - Hello, world!
solid = "hello_logs"
solid_definition = "hello_logs"
step_key = "hello_logs.compute"''')
def test_papertrail_logger():
with mock.patch("logging.handlers.SysLogHandler.emit") as emit:
result = execute_pipeline(
hello_pipeline,
{
"loggers": {
"console": {"config": {"log_level": "INFO"}},
"papertrail": {
"config": {
"log_level": "INFO",
"name": "hello_pipeline",
"papertrail_address": "127.0.0.1",
"papertrail_port": 12345,
}
},
}
},
)
log_record = emit.call_args_list[0][0][0]
assert isinstance(log_record, logging.LogRecord)
assert log_record.name == "hello_pipeline"
assert log_record.levelname == "INFO"
assert log_record.msg == "system - {run_id} - hello_logs.compute - Hello, world!".format(
run_id=result.run_id
)
def test_backcompat_no_additional_warn_text():
with mock.patch('warnings.warn') as warn_mock:
assert is_new(old_flag=False,
include_additional_warn_txt=False) is True
warn_mock.assert_called_once_with(
'"old_flag" is deprecated and will be removed in 0.9.0, use "new_flag" instead.',
stacklevel=3,
)
def test_successful_host_dagit_ui():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory() as temp_dir:
recon_repo = ReconstructableRepository.from_yaml(
file_relative_path(__file__, './repository.yaml')
)
host_dagit_ui_with_reconstructable_repo(
storage_fallback=temp_dir, recon_repo=recon_repo, host=None, port=2343
)
def test_dataproc_resource():
"""Tests dataproc cluster creation/deletion. Requests are captured by the responses library, so
no actual HTTP requests are made here.
Note that inspecting the HTTP requests can be useful for debugging, which can be done by adding:
import httplib2
httplib2.debuglevel = 4
"""
with mock.patch("httplib2.Http", new=HttpSnooper):
pipeline = PipelineDefinition(
name="test_dataproc_resource",
solid_defs=[dataproc_solid],
mode_defs=[ModeDefinition(resource_defs={"dataproc": dataproc_resource})],
)
result = execute_pipeline(
pipeline,
{
"solids": {
"dataproc_solid": {
"config": {
"job_config": {
"projectId": PROJECT_ID,
"region": REGION,
"job": {
"reference": {"projectId": PROJECT_ID},
"placement": {"clusterName": CLUSTER_NAME},
"hiveJob": {"queryList": {"queries": ["SHOW DATABASES"]}},
},
},
"job_scoped_cluster": True,
}
}
},
"resources": {
"dataproc": {
"config": {
"projectId": PROJECT_ID,
"clusterName": CLUSTER_NAME,
"region": REGION,
"cluster_config": {
"softwareConfig": {
"properties": {
# Create a single-node cluster
# This needs to be the string "true" when
# serialized, not a boolean true
"dataproc:dataproc.allow.zero.workers": "true"
}
}
},
}
}
},
},
)
assert result.success
def test_successful_host_dagit_ui_from_legacy_repository():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
workspace = load_workspace_from_yaml_path(
file_relative_path(__file__, './workspace.yaml'))
host_dagit_ui_with_workspace(storage_fallback=temp_dir,
workspace=workspace,
host=None,
port=2343)
def test_successful_host_dagit_ui():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
handle = ExecutionTargetHandle.for_repo_yaml(
file_relative_path(__file__, './repository.yaml'))
host_dagit_ui_with_execution_handle(storage_fallback=temp_dir,
handle=handle,
host=None,
port=2343)
def test_load_legacy_repository_yaml():
with mock.patch('warnings.warn') as warn_mock:
workspace = load_workspace_from_yaml_path(
file_relative_path(__file__, 'legacy_repository.yaml'))
assert isinstance(workspace, Workspace)
assert len(workspace.repository_location_handles) == 1
warn_mock.assert_called_once_with(
'You are using the legacy repository yaml format. Please update your file '
'to abide by the new workspace file format.')
def test_warnings_execute_pipeline_iterator():
assert execute_pipeline(noop_pipeline).success
with mock.patch('warnings.warn') as warn_mock:
list(execute_pipeline_iterator(noop_pipeline, environment_dict={}))
warn_mock.assert_called_with(
'"environment_dict" is deprecated and will be removed in 0.9.0, '
'use "run_config" instead.',
stacklevel=5,
)
def test_successful_host_dagit_ui_from_legacy_repository():
with mock.patch("gevent.pywsgi.WSGIServer"), seven.TemporaryDirectory(
) as temp_dir:
instance = DagsterInstance.get(temp_dir)
with load_workspace_from_yaml_paths(
[file_relative_path(__file__, "./workspace.yaml")]) as workspace:
host_dagit_ui_with_workspace(instance=instance,
workspace=workspace,
host=None,
port=2343,
path_prefix="")
def test_port_collision():
def _raise_os_error():
raise OSError('Address already in use')
with mock.patch('gevent.pywsgi.WSGIServer',
new=_define_mock_server(_raise_os_error)):
handle = ExecutionTargetHandle.for_repo_yaml(
script_relative_path('./repository.yaml'))
with pytest.raises(Exception) as exc_info:
host_dagit_ui(handle=handle, host=None, port=2343)
assert 'Another process ' in str(exc_info.value)
def test_cache_file_from_s3_step_one_one():
boto_s3 = mock.MagicMock()
# mock.patch is difficult to get right and requires monkeypatching of global artifacts
with get_temp_dir() as temp_dir, mock.patch(
file_cache_folder.__module__ + '.file_cache_folder', new=lambda: temp_dir
), mock.patch('boto3.client', new=lambda *_args, **_kwargs: boto_s3):
@solid
def emit_value(_):
return {'bucket': 'some-bucket', 'key': 'some-key'}
@pipeline
def pipe():
return cache_file_from_s3(emit_value())
execute_pipeline(pipe)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_unknown_error():
class AnException(Exception):
pass
def _raise_custom_error():
raise AnException('foobar')
with mock.patch('gevent.pywsgi.WSGIServer',
new=_define_mock_server(_raise_custom_error)):
handle = ExecutionTargetHandle.for_repo_yaml(
script_relative_path('./repository.yaml'))
with pytest.raises(AnException):
host_dagit_ui(handle=handle, host=None, port=2343)
def test_successful_host_dagit_ui():
with mock.patch('gevent.pywsgi.WSGIServer'):
handle = ExecutionTargetHandle.for_repo_yaml(
script_relative_path('./repository.yaml'))
host_dagit_ui(
log=False,
log_dir=None,
schedule_dir=None,
handle=handle,
use_sync=True,
host=None,
port=2343,
)
def test_valid_path_prefix():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
workspace = load_workspace_from_yaml_paths(
[file_relative_path(__file__, './workspace.yaml')])
host_dagit_ui_with_workspace(
storage_fallback=temp_dir,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix='/dagster-path',
)
def test_successful_host_dagit_ui_from_multiple_workspace_files():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
workspace = load_workspace_from_yaml_paths([
file_relative_path(__file__, './workspace.yaml'),
file_relative_path(__file__, './override.yaml'),
])
host_dagit_ui_with_workspace(storage_fallback=temp_dir,
workspace=workspace,
host=None,
port=2343,
path_prefix='')
def test_successful_host_dagit_ui_from_legacy_repository():
with mock.patch('gevent.pywsgi.WSGIServer'), seven.TemporaryDirectory(
) as temp_dir:
instance = DagsterInstance.get(temp_dir)
workspace = load_workspace_from_yaml_paths(
[file_relative_path(__file__, './workspace.yaml')],
UserProcessApi.CLI,
)
host_dagit_ui_with_workspace(instance=instance,
workspace=workspace,
host=None,
port=2343,
path_prefix='')
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
'boto3.client', new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir)}),
input_values={'s3_coord': {'bucket': 'some-bucket', 'key': 'some-key'}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
"boto3.client", new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({"file_cache": FSFileCache(temp_dir)}),
input_values={"s3_coord": {"bucket": "some-bucket", "key": "some-key"}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, "some-key"))
def test_unknown_error():
class AnException(Exception):
pass
def _raise_custom_error():
raise AnException('foobar')
with mock.patch(
'gevent.pywsgi.WSGIServer', new=_define_mock_server(_raise_custom_error)
), seven.TemporaryDirectory() as temp_dir:
handle = ExecutionTargetHandle.for_repo_yaml(
file_relative_path(__file__, './repository.yaml')
)
with pytest.raises(AnException):
host_dagit_ui(storage_fallback=temp_dir, handle=handle, host=None, port=2343)
def test_valid_path_prefix():
with mock.patch("gevent.pywsgi.WSGIServer"), seven.TemporaryDirectory(
) as temp_dir:
instance = DagsterInstance.get(temp_dir)
with load_workspace_from_yaml_paths(
[file_relative_path(__file__, "./workspace.yaml")]) as workspace:
host_dagit_ui_with_workspace(
instance=instance,
workspace=workspace,
host=None,
port=2343,
port_lookup=False,
path_prefix="/dagster-path",
)
def test_port_collision():
def _raise_os_error():
raise OSError('Address already in use')
with mock.patch(
'gevent.pywsgi.WSGIServer', new=_define_mock_server(_raise_os_error)
), seven.TemporaryDirectory() as temp_dir:
handle = ExecutionTargetHandle.for_repo_yaml(
file_relative_path(__file__, './repository.yaml')
)
with pytest.raises(Exception) as exc_info:
host_dagit_ui(
storage_fallback=temp_dir, handle=handle, host=None, port=2343, port_lookup=False
)
assert 'another instance of dagit ' in str(exc_info.value)
