def test_s3_io(monkeypatch, without_aws_env):
import moto
from numpy import s_
url = "s3://bucket/file.txt"
bucket, _ = s3_url_parse(url)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
with moto.mock_s3():
s3 = s3_client(region_name='kk')
s3.create_bucket(Bucket=bucket)
assert s3_dump(b"33", url, s3=s3) is True
assert s3_fetch(url, s3=s3) == b"33"
meta = s3_head_object(url, s3=s3)
assert meta is not None
assert 'LastModified' in meta
assert 'ContentLength' in meta
assert 'ETag' in meta
assert meta['ContentLength'] == 2
assert s3_head_object(url + '-nosuch', s3=s3) is None
assert s3_dump(b"0123456789ABCDEF", url, s3=s3) is True
assert s3_fetch(url, range=s_[:4], s3=s3) == b"0123"
assert s3_fetch(url, range=s_[3:8], s3=s3) == b"34567"
with pytest.raises(ValueError):
s3_fetch(url, range=s_[::2], s3=s3)
def test_save_blob_s3_direct(blob, monkeypatch):
region_name = "us-west-2"
blob2 = blob + blob
url = "s3://bucket/file.txt"
url2 = "s3://bucket/file-2.txt"
bucket, _ = s3_url_parse(url)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
with moto.mock_s3():
s3 = s3_client(region_name=region_name)
s3.create_bucket(Bucket=bucket)
assert _save_blob_to_s3(blob, url,
region_name=region_name) == (url, True)
assert _save_blob_to_s3(blob2, url2,
region_name=region_name) == (url2, True)
bb1 = s3_fetch(url, s3=s3)
bb2 = s3_fetch(url2, s3=s3)
if isinstance(blob, str):
bb1 = bb1.decode("utf8")
bb2 = bb2.decode("utf8")
assert bb1 == blob
assert bb2 == blob2
assert _save_blob_to_s3(
"",
"s3://not-a-bucket/f.txt") == ("s3://not-a-bucket/f.txt", False)
def test_save_blob_s3(blob, monkeypatch, dask_client):
region_name = "us-west-2"
blob2 = blob + blob
dask_blob = dask.delayed(blob)
dask_blob2 = dask.delayed(blob2)
url = "s3://bucket/file.txt"
url2 = "s3://bucket/file-2.txt"
bucket, _ = s3_url_parse(url)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
with moto.mock_s3():
s3 = s3_client(region_name=region_name)
s3.create_bucket(Bucket=bucket)
rr = save_blob_to_s3(dask_blob, url, region_name=region_name)
assert rr.compute() == (url, True)
rr = save_blob_to_s3(dask_blob2, url2, region_name=region_name)
assert dask_client.compute(rr).result() == (url2, True)
bb1 = s3_fetch(url, s3=s3)
bb2 = s3_fetch(url2, s3=s3)
if isinstance(blob, str):
bb1 = bb1.decode("utf8")
bb2 = bb2.decode("utf8")
assert bb1 == blob
assert bb2 == blob2
def test_s3_io(monkeypatch, without_aws_env):
import moto
from numpy import s_
url = "s3://bucket/file.txt"
bucket, _ = s3_url_parse(url)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret")
with moto.mock_s3():
s3 = s3_client(region_name='kk')
s3.create_bucket(Bucket=bucket)
assert s3_dump(b"33", url, s3=s3) is True
assert s3_fetch(url, s3=s3) == b"33"
assert s3_dump(b"0123456789ABCDEF", url, s3=s3) is True
assert s3_fetch(url, range=s_[:4], s3=s3) == b"0123"
assert s3_fetch(url, range=s_[3:8], s3=s3) == b"34567"
with pytest.raises(ValueError):
s3_fetch(url, range=s_[::2], s3=s3)