def unittest_for_aws_mode_def(s3_session):
return ModeDefinition.from_resources({
'file_cache':
S3FileCache('file-cache-bucket', 'file-cache', s3_session),
's3':
s3_session,
})
def unittest_for_aws_mode_def(s3_session):
return ModeDefinition.from_resources({
"file_cache":
S3FileCache("file-cache-bucket", "file-cache", s3_session),
"s3":
s3_session,
})
def test_mode_from_resources():
@solid(required_resource_keys={'three'})
def ret_three(context):
return context.resources.three
@pipeline(
name='takes a mode', mode_defs=[ModeDefinition.from_resources({'three': 3}, name='three')]
)
def pipeline_def():
return ret_three()
assert execute_pipeline(pipeline_def).result_for_solid('ret_three').output_value() == 3
def test_mode_from_resources():
@solid
def ret_three(context):
return context.resources.three
@pipeline(
name='takes a mode', mode_defs=[ModeDefinition.from_resources({'three': 3}, name='three')]
)
def pipeline_def():
return ret_three() # pylint: disable=no-value-for-parameter
assert execute_pipeline(pipeline_def).result_for_solid('ret_three').output_value() == 3
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
"boto3.client", new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({"file_cache": FSFileCache(temp_dir)}),
input_values={"s3_coord": {"bucket": "some-bucket", "key": "some-key"}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, "some-key"))
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
'boto3.client', new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir)}),
input_values={'s3_coord': {'bucket': 'some-bucket', 'key': 'some-key'}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def unittest_for_local_mode_def(temp_dir, s3_session):
return ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir), 's3': s3_session})
