def test_s3_basics(without_aws_env): from numpy import s_ from botocore.credentials import ReadOnlyCredentials assert s3_url_parse('s3://bucket/key') == ('bucket', 'key') assert s3_url_parse('s3://bucket/key/') == ('bucket', 'key/') assert s3_url_parse('s3://bucket/k/k/key') == ('bucket', 'k/k/key') with pytest.raises(ValueError): s3_url_parse("file://some/path") assert s3_fmt_range((0, 3)) == "bytes=0-2" assert s3_fmt_range(s_[4:10]) == "bytes=4-9" assert s3_fmt_range(s_[:10]) == "bytes=0-9" assert s3_fmt_range(None) is None for bad in (s_[10:], s_[-2:3], s_[:-3], (-1, 3), (3, -1), s_[1:100:3]): with pytest.raises(ValueError): s3_fmt_range(bad) creds = ReadOnlyCredentials('fake-key', 'fake-secret', None) assert str(s3_client( region_name='kk')._endpoint) == 's3(https://s3.kk.amazonaws.com)' assert str( s3_client(region_name='kk', use_ssl=False)._endpoint) == 's3(http://s3.kk.amazonaws.com)' s3 = s3_client(region_name='us-west-2', creds=creds) assert s3 is not None
def test_s3_io(monkeypatch, without_aws_env): import moto from numpy import s_ url = "s3://bucket/file.txt" bucket, _ = s3_url_parse(url) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret") with moto.mock_s3(): s3 = s3_client(region_name='kk') s3.create_bucket(Bucket=bucket) assert s3_dump(b"33", url, s3=s3) is True assert s3_fetch(url, s3=s3) == b"33" meta = s3_head_object(url, s3=s3) assert meta is not None assert 'LastModified' in meta assert 'ContentLength' in meta assert 'ETag' in meta assert meta['ContentLength'] == 2 assert s3_head_object(url + '-nosuch', s3=s3) is None assert s3_dump(b"0123456789ABCDEF", url, s3=s3) is True assert s3_fetch(url, range=s_[:4], s3=s3) == b"0123" assert s3_fetch(url, range=s_[3:8], s3=s3) == b"34567" with pytest.raises(ValueError): s3_fetch(url, range=s_[::2], s3=s3)
def test_save_blob_s3_direct(blob, monkeypatch): region_name = "us-west-2" blob2 = blob + blob url = "s3://bucket/file.txt" url2 = "s3://bucket/file-2.txt" bucket, _ = s3_url_parse(url) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret") with moto.mock_s3(): s3 = s3_client(region_name=region_name) s3.create_bucket(Bucket=bucket) assert _save_blob_to_s3(blob, url, region_name=region_name) == (url, True) assert _save_blob_to_s3(blob2, url2, region_name=region_name) == (url2, True) bb1 = s3_fetch(url, s3=s3) bb2 = s3_fetch(url2, s3=s3) if isinstance(blob, str): bb1 = bb1.decode("utf8") bb2 = bb2.decode("utf8") assert bb1 == blob assert bb2 == blob2 assert _save_blob_to_s3( "", "s3://not-a-bucket/f.txt") == ("s3://not-a-bucket/f.txt", False)
def test_save_blob_s3(blob, monkeypatch, dask_client): region_name = "us-west-2" blob2 = blob + blob dask_blob = dask.delayed(blob) dask_blob2 = dask.delayed(blob2) url = "s3://bucket/file.txt" url2 = "s3://bucket/file-2.txt" bucket, _ = s3_url_parse(url) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret") with moto.mock_s3(): s3 = s3_client(region_name=region_name) s3.create_bucket(Bucket=bucket) rr = save_blob_to_s3(dask_blob, url, region_name=region_name) assert rr.compute() == (url, True) rr = save_blob_to_s3(dask_blob2, url2, region_name=region_name) assert dask_client.compute(rr).result() == (url2, True) bb1 = s3_fetch(url, s3=s3) bb2 = s3_fetch(url2, s3=s3) if isinstance(blob, str): bb1 = bb1.decode("utf8") bb2 = bb2.decode("utf8") assert bb1 == blob assert bb2 == blob2
def test_s3_io(monkeypatch, without_aws_env): import moto from numpy import s_ url = "s3://bucket/file.txt" bucket, _ = s3_url_parse(url) monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake-key-id") monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake-secret") with moto.mock_s3(): s3 = s3_client(region_name='kk') s3.create_bucket(Bucket=bucket) assert s3_dump(b"33", url, s3=s3) is True assert s3_fetch(url, s3=s3) == b"33" assert s3_dump(b"0123456789ABCDEF", url, s3=s3) is True assert s3_fetch(url, range=s_[:4], s3=s3) == b"0123" assert s3_fetch(url, range=s_[3:8], s3=s3) == b"34567" with pytest.raises(ValueError): s3_fetch(url, range=s_[::2], s3=s3)