def test_s3_put_data(s3_bucket):
s3 = aws.S3()
key = s3.put(
schema.Data(metadata=schema.Metadata(timestamp=0), data='test'))
res = schema.Data.from_json(s3_bucket.Object(key).get()['Body'].read())
assert res.data == 'test'
def test_s3fs_open_write(s3_bucket):
s3 = aws.S3(access_path='data')
with s3.fs.open('test.txt', 'w') as f:
f.write('test')
assert s3_bucket.Object('data/test.txt').get()['Body'].read() == b'test'
def test_s3_key(path, expected_key_pattern):
s3 = aws.S3(access_path='abc/123')
key = s3.put(schema.Data(metadata=schema.Metadata(timestamp=1234),
data=''),
path=path)
assert re.fullmatch(expected_key_pattern, key)
def test_s3_get_data_json(s3_bucket):
s3_bucket.Object('/data/test.json').put(
Body='{"hello":"world"}'.encode('utf-8'))
s3 = aws.S3(access_path='/data')
test_json = s3.get('test.json').json()
assert test_json['hello'] == 'world'
def test_s3_put_raw_data(s3_bucket):
s3 = aws.S3()
input_bytes = 'some bytes'
ext = 'txt'
key = s3.put_raw(input_bytes, path='', ext=ext)
res = s3_bucket.Object(key).get()['Body'].read().decode('utf-8')
assert res == input_bytes
assert ext in key
def test_s3_default_overrides():
s3 = aws.S3(access_path='/abc', bucket='myBucket')
assert s3.access_path == '/abc' and \
s3.bucket == 'myBucket'
def test_s3_default():
s3 = aws.S3()
assert s3.access_path == environ.get("ACCESS_PATH") and \
s3.bucket == environ.get('DATALAKE')
def test_s3fs_exists(s3_bucket):
s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8'))
s3 = aws.S3(access_path='data')
assert s3.fs.exists('test.txt') is True
def test_s3fs_open_read(s3_bucket):
s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8'))
s3 = aws.S3(access_path='data')
with s3.fs.open('test.txt', 'r') as f:
assert f.read() == 'test'
def test_s3_get_absolute_path_data(s3_bucket):
s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8'))
s3 = aws.S3(access_path='/data')
test_text = s3.get('/data/test.txt').raw
assert b'test' == test_text
def test_s3_get_empty():
s3 = aws.S3(access_path='/data')
res = s3.get('test.txt')
assert res.raw is None and \
'NoSuchKey' in str(res.error)
def test_s3_put_raw_data_with_path(s3_bucket):
s3 = aws.S3(access_path='')
input_bytes = 'some_bytes'
input_key = 'test_file.txt'
key = s3.put_raw(input_bytes, key=input_key, path='test')
assert key == f'test/{input_key}'
def test_s3_put_raw_data_access_path_empty(s3_bucket):
s3 = aws.S3(access_path='')
input_bytes = 'some_bytes'
input_key = 'test_file.txt'
key = s3.put_raw(input_bytes, key=input_key)
assert key == input_key
def test_s3_put_raw_data_none(s3_bucket):
s3 = aws.S3()
input_bytes = None
ext = 'txt'
key = s3.put_raw(input_bytes, path='', ext=ext)
assert key is None
