def test_s3_put_data(s3_bucket): s3 = aws.S3() key = s3.put( schema.Data(metadata=schema.Metadata(timestamp=0), data='test')) res = schema.Data.from_json(s3_bucket.Object(key).get()['Body'].read()) assert res.data == 'test'
def test_s3fs_open_write(s3_bucket): s3 = aws.S3(access_path='data') with s3.fs.open('test.txt', 'w') as f: f.write('test') assert s3_bucket.Object('data/test.txt').get()['Body'].read() == b'test'
def test_s3_key(path, expected_key_pattern): s3 = aws.S3(access_path='abc/123') key = s3.put(schema.Data(metadata=schema.Metadata(timestamp=1234), data=''), path=path) assert re.fullmatch(expected_key_pattern, key)
def test_s3_get_data_json(s3_bucket): s3_bucket.Object('/data/test.json').put( Body='{"hello":"world"}'.encode('utf-8')) s3 = aws.S3(access_path='/data') test_json = s3.get('test.json').json() assert test_json['hello'] == 'world'
def test_s3_put_raw_data(s3_bucket): s3 = aws.S3() input_bytes = 'some bytes' ext = 'txt' key = s3.put_raw(input_bytes, path='', ext=ext) res = s3_bucket.Object(key).get()['Body'].read().decode('utf-8') assert res == input_bytes assert ext in key
def test_s3_default_overrides(): s3 = aws.S3(access_path='/abc', bucket='myBucket') assert s3.access_path == '/abc' and \ s3.bucket == 'myBucket'
def test_s3_default(): s3 = aws.S3() assert s3.access_path == environ.get("ACCESS_PATH") and \ s3.bucket == environ.get('DATALAKE')
def test_s3fs_exists(s3_bucket): s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8')) s3 = aws.S3(access_path='data') assert s3.fs.exists('test.txt') is True
def test_s3fs_open_read(s3_bucket): s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8')) s3 = aws.S3(access_path='data') with s3.fs.open('test.txt', 'r') as f: assert f.read() == 'test'
def test_s3_get_absolute_path_data(s3_bucket): s3_bucket.Object('/data/test.txt').put(Body='test'.encode('utf-8')) s3 = aws.S3(access_path='/data') test_text = s3.get('/data/test.txt').raw assert b'test' == test_text
def test_s3_get_empty(): s3 = aws.S3(access_path='/data') res = s3.get('test.txt') assert res.raw is None and \ 'NoSuchKey' in str(res.error)
def test_s3_put_raw_data_with_path(s3_bucket): s3 = aws.S3(access_path='') input_bytes = 'some_bytes' input_key = 'test_file.txt' key = s3.put_raw(input_bytes, key=input_key, path='test') assert key == f'test/{input_key}'
def test_s3_put_raw_data_access_path_empty(s3_bucket): s3 = aws.S3(access_path='') input_bytes = 'some_bytes' input_key = 'test_file.txt' key = s3.put_raw(input_bytes, key=input_key) assert key == input_key
def test_s3_put_raw_data_none(s3_bucket): s3 = aws.S3() input_bytes = None ext = 'txt' key = s3.put_raw(input_bytes, path='', ext=ext) assert key is None