def test_cache_file_from_s3_step_three_fake(snapshot): s3 = boto3.client('s3') s3.create_bucket(Bucket='some-bucket') s3.put_object(Bucket='some-bucket', Key='some-key', Body=b'foo') with get_temp_dir() as temp_dir: execute_solid( cache_file_from_s3, unittest_for_local_mode_def(temp_dir, s3), input_values={ 's3_coord': { 'bucket': 'some-bucket', 'key': 'some-key' } }, ) target_file = os.path.join(temp_dir, 'some-key') assert os.path.exists(target_file) with open(target_file, 'rb') as ff: assert ff.read() == b'foo' snapshot.assert_match({ 'some-bucket': { k: s3.get_object(Bucket='some-bucket', Key=k)['Body'].read() for k in [ obj['Key'] for obj in s3.list_objects(Bucket='some-bucket')['Contents'] ] } })
def test_single_solid_type_checking_output_error(): @lambda_solid(output_def=OutputDefinition(Int)) def return_string(): return 'ksjdfkjd' with pytest.raises(DagsterTypeCheckError): execute_solid(return_string)
def test_cache_file_from_s3_step_three_fake(snapshot): # https://github.com/spulec/moto/issues/3292 s3 = boto3.client("s3", region_name="us-east-1") s3.create_bucket(Bucket="some-bucket") s3.put_object(Bucket="some-bucket", Key="some-key", Body=b"foo") with get_temp_dir() as temp_dir: execute_solid( cache_file_from_s3, unittest_for_local_mode_def(temp_dir, s3), input_values={ "s3_coord": { "bucket": "some-bucket", "key": "some-key" } }, ) target_file = os.path.join(temp_dir, "some-key") assert os.path.exists(target_file) with open(target_file, "rb") as ff: assert ff.read() == b"foo" snapshot.assert_match({ "some-bucket": { k: s3.get_object(Bucket="some-bucket", Key=k)["Body"].read() for k in [ obj["Key"] for obj in s3.list_objects(Bucket="some-bucket")["Contents"] ] } })
def test_single_solid_with_context_config(): @resource(config_schema=Field(Int, is_required=False, default_value=2)) def num_resource(init_context): return init_context.resource_config ran = {"count": 0} @solid(required_resource_keys={"num"}) def check_context_config_for_two(context): assert context.resources.num == 2 ran["count"] += 1 result = execute_solid( check_context_config_for_two, run_config={"resources": { "num": { "config": 2 } }}, mode_def=ModeDefinition(resource_defs={"num": num_resource}), ) assert result.success assert ran["count"] == 1 result = execute_solid( check_context_config_for_two, mode_def=ModeDefinition(resource_defs={"num": num_resource})) assert result.success assert ran["count"] == 2
def test_composites(): @lambda_solid def hello(): return 'hello' @composite_solid def hello_composite(): return hello() result = execute_solid(hello) assert result.success assert result.output_value() == 'hello' assert result.output_values == {'result': 'hello'} result = execute_solid(hello_composite) assert result.success assert result.output_value() == 'hello' assert result.output_values == {'result': 'hello'} assert result.output_values_for_solid('hello') == {'result': 'hello'} assert result.output_value_for_handle('hello') == 'hello' nested_result = result.result_for_solid('hello') assert nested_result.success assert nested_result.output_value() == 'hello' assert len(result.solid_result_list) == 1 assert nested_result.output_values == {'result': 'hello'} with pytest.raises( DagsterInvariantViolationError, match=re.escape( 'Tried to get result for solid \'goodbye\' in \'hello_composite\'. No such top level ' 'solid' ), ): _ = result.result_for_solid('goodbye')
def test_cache_file_from_s3_step_two_use_config(): boto_s3 = mock.MagicMock() with get_temp_dir() as temp_dir, mock.patch( 'boto3.client', new=lambda *_args, **_kwargs: boto_s3): execute_solid( cache_file_from_s3, ModeDefinition(resource_defs={'file_cache': fs_file_cache}), environment_dict={ 'resources': { 'file_cache': { 'config': { 'target_folder': temp_dir } } }, 'solids': { 'cache_file_from_s3': { 'inputs': { 's3_coord': { 'bucket': 'some-bucket', 'key': 'some-key' } } } }, }, ) assert boto_s3.download_file.call_count == 1 assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_single_solid_with_context_config(): ran = {'check_context_config_for_two': 0} @solid def check_context_config_for_two(info): assert info.resources == 2 ran['check_context_config_for_two'] += 1 pipeline_def = PipelineDefinition( solids=[check_context_config_for_two], context_definitions={ 'test_context': PipelineContextDefinition( config_field=Field(Int, is_optional=True, default_value=2), context_fn=lambda info: ExecutionContext(resources=info.config), ) }, ) result = execute_solid( pipeline_def, 'check_context_config_for_two', environment={'context': {'test_context': {'config': 2}}}, ) assert result.success assert ran['check_context_config_for_two'] == 1 result = execute_solid(pipeline_def, 'check_context_config_for_two') assert result.success assert ran['check_context_config_for_two'] == 2
def test_composites(): @lambda_solid def hello(): return "hello" @composite_solid def hello_composite(): return hello() result = execute_solid(hello) assert result.success assert result.output_value() == "hello" assert result.output_values == {"result": "hello"} result = execute_solid(hello_composite) assert result.success assert result.output_value() == "hello" assert result.output_values == {"result": "hello"} assert result.output_values_for_solid("hello") == {"result": "hello"} assert result.output_value_for_handle("hello") == "hello" nested_result = result.result_for_solid("hello") assert nested_result.success assert nested_result.output_value() == "hello" assert len(result.solid_result_list) == 1 assert nested_result.output_values == {"result": "hello"} with pytest.raises( DagsterInvariantViolationError, match=re.escape( "Tried to get result for solid 'goodbye' in 'hello_composite'. No such top level " "solid"), ): _ = result.result_for_solid("goodbye")
def test_single_solid_with_context_config(): @resource(config_field=Field(Int, is_optional=True, default_value=2)) def num_resource(init_context): return init_context.resource_config ran = {'count': 0} @solid def check_context_config_for_two(context): assert context.resources.num == 2 ran['count'] += 1 result = execute_solid( check_context_config_for_two, environment_dict={'resources': {'num': {'config': 2}}}, mode_def=ModeDefinition(resource_defs={'num': num_resource}), ) assert result.success assert ran['count'] == 1 result = execute_solid( check_context_config_for_two, mode_def=ModeDefinition(resource_defs={'num': num_resource}) ) assert result.success assert ran['count'] == 2
def test_single_solid_type_checking_output_error(): @lambda_solid(output=OutputDefinition(types.Int)) def return_string(): return 'ksjdfkjd' pipeline_def = PipelineDefinition(solids=[return_string]) with pytest.raises(DagsterInvariantViolationError): execute_solid(pipeline_def, 'return_string')
def test_failing_solid_in_isolation(): class ThisException(Exception): pass @lambda_solid def throw_an_error(): raise ThisException("nope") with pytest.raises(ThisException) as e_info: execute_solid(throw_an_error) assert isinstance(e_info.value, ThisException)
def test_single_solid_error(): class SomeError(Exception): pass @lambda_solid def throw_error(): raise SomeError() with pytest.raises(SomeError) as e_info: execute_solid(throw_error) assert isinstance(e_info.value, SomeError)
def test_failing_solid_execute_solid(): class ThisException(Exception): pass @lambda_solid def throw_an_error(): raise ThisException('nope') pipeline_def = PipelineDefinition(solids=[throw_an_error]) with pytest.raises(ThisException): execute_solid(pipeline_def, 'throw_an_error')
def test_single_solid_error(): class SomeError(Exception): pass @lambda_solid def throw_error(): raise SomeError() pipeline_def = PipelineDefinition(solids=[throw_error]) with pytest.raises(SomeError): execute_solid(pipeline_def, 'throw_error')
def test_another_new_solid(capsys): assert another_configured_example.name == "another_configured_example" execute_solid( another_configured_example, None, None, None, {"solids": {"another_configured_example": {"config": 6}}}, ) captured = capsys.readouterr() assert captured.err.count("wheaties") == 6
def test_failing_solid_in_isolation(): class ThisException(Exception): pass @lambda_solid def throw_an_error(): raise ThisException('nope') with pytest.raises(DagsterExecutionStepExecutionError) as e_info: execute_solid(throw_an_error) assert isinstance(e_info.value.__cause__, ThisException)
def test_single_solid_error(): class SomeError(Exception): pass @lambda_solid def throw_error(): raise SomeError() with pytest.raises(DagsterExecutionStepExecutionError) as e_info: execute_solid(throw_error) assert isinstance(e_info.value.__cause__, SomeError)
def test_cache_file_from_s3_step_three_mock(): s3_session = mock.MagicMock() with get_temp_dir() as temp_dir: execute_solid( cache_file_from_s3, unittest_for_local_mode_def(temp_dir, s3_session), input_values={'s3_coord': {'bucket': 'some-bucket', 'key': 'some-key'}}, ) assert s3_session.download_file.call_count == 1 assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_single_solid_error(): class SomeError(Exception): pass @lambda_solid def throw_error(): raise SomeError() pipeline_def = PipelineDefinition(solid_defs=[throw_error]) with pytest.raises(DagsterExecutionStepExecutionError) as e_info: execute_solid(pipeline_def, 'throw_error') assert isinstance(e_info.value.__cause__, SomeError)
def test_failing_solid_execute_solid(): class ThisException(Exception): pass @lambda_solid def throw_an_error(): raise ThisException('nope') pipeline_def = PipelineDefinition(solid_defs=[throw_an_error]) with pytest.raises(DagsterExecutionStepExecutionError) as e_info: execute_solid(pipeline_def, 'throw_an_error') assert isinstance(e_info.value.__cause__, ThisException)
def test_cache_file_from_s3_step_two_skip_config(): boto_s3 = mock.MagicMock() with get_temp_dir() as temp_dir, mock.patch( "boto3.client", new=lambda *_args, **_kwargs: boto_s3 ): execute_solid( cache_file_from_s3, ModeDefinition.from_resources({"file_cache": FSFileCache(temp_dir)}), input_values={"s3_coord": {"bucket": "some-bucket", "key": "some-key"}}, ) assert boto_s3.download_file.call_count == 1 assert os.path.exists(os.path.join(temp_dir, "some-key"))
def test_cache_file_from_s3_step_one_two(): boto_s3 = mock.MagicMock() # mock.patch is difficult to get right and requires monkeypatching of global artifacts with get_temp_dir() as temp_dir, mock.patch( file_cache_folder.__module__ + ".file_cache_folder", new=lambda: temp_dir ), mock.patch("boto3.client", new=lambda *_args, **_kwargs: boto_s3): execute_solid( cache_file_from_s3, input_values=dict(s3_coord={"bucket": "some-bucket", "key": "some-key"}), ) assert boto_s3.download_file.call_count == 1 assert os.path.exists(os.path.join(temp_dir, "some-key"))
def test_cache_file_from_s3_step_two_skip_config(): boto_s3 = mock.MagicMock() with get_temp_dir() as temp_dir, mock.patch( 'boto3.client', new=lambda *_args, **_kwargs: boto_s3 ): execute_solid( cache_file_from_s3, ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir)}), input_values={'s3_coord': {'bucket': 'some-bucket', 'key': 'some-key'}}, ) assert boto_s3.download_file.call_count == 1 assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_single_solid_with_bad_inputs(): @lambda_solid(input_defs=[ InputDefinition("num_one", int), InputDefinition("num_two", int) ]) def add_solid(num_one, num_two): return num_one + num_two result = execute_solid( add_solid, input_values={ "num_one": 2, "num_two": "three" }, run_config={ "loggers": { "console": { "config": { "log_level": "DEBUG" } } } }, raise_on_error=False, ) assert not result.success assert result.failure_data.error.cls_name == "DagsterTypeCheckDidNotPass" assert ('Type check failed for step input "num_two" - expected type "Int"' in result.failure_data.error.message)
def test_composite_with_no_output_mappings(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) @composite_solid def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) res = execute_solid(diamond_composite) assert res.success assert res.output_values == {} with pytest.raises( DagsterInvariantViolationError, match=re.escape( 'Output \'result\' not defined in composite solid \'diamond_composite\': no output ' 'mappings were defined. If you were expecting this output to be present, you may be ' 'missing an output_mapping from an inner solid to its enclosing composite solid.' ), ): _ = res.output_value() assert len(res.solid_result_list) == 5
def test_cache_file_from_s3_step_four(snapshot): s3 = boto3.client('s3') s3.create_bucket(Bucket='source-bucket') s3.create_bucket(Bucket='file-cache-bucket') s3.put_object(Bucket='source-bucket', Key='source-file', Body=b'foo') solid_result = execute_solid( cache_file_from_s3, unittest_for_aws_mode_def(s3), input_values={ 's3_coord': { 'bucket': 'source-bucket', 'key': 'source-file' } }, ) assert solid_result.output_value( ).path_desc == 's3://file-cache-bucket/file-cache/source-file' file_cache_obj = s3.get_object(Bucket='file-cache-bucket', Key='file-cache/source-file') assert file_cache_obj['Body'].read() == b'foo' snapshot.assert_match({ 'file-cache-bucket': { k: s3.get_object(Bucket='file-cache-bucket', Key=k)['Body'].read() for k in [ obj['Key'] for obj in s3.list_objects( Bucket='file-cache-bucket')['Contents'] ] } })
def test_single_solid_with_multiple_inputs(): @lambda_solid(input_defs=[ InputDefinition(name='num_one'), InputDefinition('num_two') ]) def add_solid(num_one, num_two): return num_one + num_two result = execute_solid( add_solid, input_values={ 'num_one': 2, 'num_two': 3 }, environment_dict={ 'loggers': { 'console': { 'config': { 'log_level': 'DEBUG' } } } }, ) assert result.success assert result.output_value() == 5
def test_cache_file_from_s3_step_four(snapshot): s3 = boto3.client("s3") s3.create_bucket(Bucket="source-bucket") s3.create_bucket(Bucket="file-cache-bucket") s3.put_object(Bucket="source-bucket", Key="source-file", Body=b"foo") solid_result = execute_solid( cache_file_from_s3, unittest_for_aws_mode_def(s3), input_values={ "s3_coord": { "bucket": "source-bucket", "key": "source-file" } }, ) assert solid_result.output_value( ).path_desc == "s3://file-cache-bucket/file-cache/source-file" file_cache_obj = s3.get_object(Bucket="file-cache-bucket", Key="file-cache/source-file") assert file_cache_obj["Body"].read() == b"foo" snapshot.assert_match({ "file-cache-bucket": { k: s3.get_object(Bucket="file-cache-bucket", Key=k)["Body"].read() for k in [ obj["Key"] for obj in s3.list_objects( Bucket="file-cache-bucket")["Contents"] ] } })
def test_single_solid_with_multiple_inputs(): @lambda_solid(input_defs=[ InputDefinition(name="num_one"), InputDefinition("num_two") ]) def add_solid(num_one, num_two): return num_one + num_two result = execute_solid( add_solid, input_values={ "num_one": 2, "num_two": 3 }, run_config={ "loggers": { "console": { "config": { "log_level": "DEBUG" } } } }, ) assert result.success assert result.output_value() == 5
def test_cache_file_from_s3_step_four(snapshot): s3_session = S3FakeSession({'source-bucket': {'source-file': b'foo'}}) s3_file_cache_session = S3FakeSession() solid_result = execute_solid( cache_file_from_s3, unittest_for_aws_mode_def(s3_file_cache_session, s3_session), input_values={ 's3_coord': { 'bucket': 'source-bucket', 'key': 'source-file' } }, ) assert solid_result.output_value( ).path_desc == 's3://file-cache-bucket/file-cache/source-file' file_cache_obj = s3_file_cache_session.get_object( Bucket='file-cache-bucket', Key='file-cache/source-file') assert file_cache_obj['Body'].read() == b'foo' # just perform a snapshot of the bucket structure as well snapshot.assert_match(s3_file_cache_session.buckets)