def test_default_config():
config = omnio.default_config()
for scheme in ["file", "http", "s3"]:
assert isinstance(config[scheme], dict)
omnio.open("tests/data/ascii.txt", 'rt', config=config)
def test_rtz():
with omnio.open('tests/data/flights-3m.csv.gz', 'rtz') as fd:
reader = csv.reader(fd)
headers = next(reader)
assert headers == [
'date', 'delay', 'distance', 'origin', 'destination'
]
data = list(reader)
assert len(data) == 231083 # number data rows
def test_read_chunk():
uri = 'http://example.com/example'
data = os.urandom(1024)
responses.add(responses.GET, uri, body=data, status=200)
with omnio.open(uri, 'rb') as infile:
chunk = infile.read(1000)
assert len(chunk) == 1000
assert chunk == data[:1000]
chunk = infile.read(1000)
assert len(chunk) == 24
assert chunk == data[1000:]
def test_w():
path = 'tests/data/w.txt'
try:
with omnio.open(path, 'wt') as fd:
fd.write('one\ntwo\nthree\n')
except Exception:
raise
else:
with open(path, 'r') as fd:
assert list(fd) == ['one\n', 'two\n', 'three\n']
finally:
os.remove(path)
def test_open_rt_utf8():
uri = 'http://example.com/example'
data = _load_test_data('utf-8')
responses.add(
responses.GET,
uri,
body=data,
status=200,
content_type='text/plain; charset=UTF-8',
)
with omnio.open(uri, 'rt') as infile:
assert infile.read() == data.decode('utf-8')
def test_closed():
uri = 'http://example.com/example'
data = b'foo bar baz'
responses.add(responses.GET, uri, body=data, status=200)
f = omnio.open(uri, 'rb')
f.close()
# none of these operations are allowed on a closed file
with pytest.raises(ValueError):
f.read()
with pytest.raises(ValueError):
next(f)
with pytest.raises(ValueError):
iter(f)
def test_read_connection_error(monkeypatch):
class Client:
def __init__(self, *args, **kwargs):
pass
def get_object(self, Bucket=None, Key=None):
raise botocore.exceptions.EndpointConnectionError(
endpoint_url="test/")
monkeypatch.setattr("boto3.client", Client)
with pytest.raises(ConnectionError):
with omnio.open("s3://bucket/key"):
pass
def test_wbj():
path = 'tests/data/wbj.txt.bz2'
data = os.urandom(1024)
try:
with omnio.open(path, 'wbj') as fd:
fd.write(data)
except Exception:
raise
else:
with open(path, 'rb') as fd:
with bz2.open(fd) as gz:
assert gz.read() == data
finally:
os.remove(path)
def test_wtz():
path = 'tests/data/wtz.txt.gz'
data = 'unicode string to be seamlessly compressed'
try:
with omnio.open(path, 'wtz') as fd:
fd.write(data)
except Exception:
raise
else:
with open(path, 'rb') as fd:
with gzip.open(fd, 'rt') as gz:
assert gz.read() == data
finally:
os.remove(path)
def test_iter():
uri = 'http://example.com/example'
data = _load_test_data('utf-8')
responses.add(
responses.GET,
uri,
body=data,
status=200,
content_type='text/plain; charset=UTF-8',
)
expect_first = (
'Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ' '᾿Αθηναῖοι,\n'.encode('utf-8')
)
expect_last = 'τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.'.encode('utf-8')
with omnio.open(uri, 'rb') as infile:
lines = list(infile)
assert len(lines) == 16
assert lines[0] == expect_first
assert lines[-1] == expect_last
def test_invalid_mode():
with pytest.raises(ValueError):
omnio.open('', 'q')
def test_invalid_combination():
# can't combine read and write
with pytest.raises(ValueError):
omnio.open('', 'rw')
# can't specify encoding in binary mode
with pytest.raises(ValueError):
omnio.open('', 'rb', encoding='utf-8')
# can't specify errors in binary mode
with pytest.raises(ValueError):
omnio.open('', 'rb', errors='strict')
# can't specify newline in binary mode
with pytest.raises(ValueError):
omnio.open('', 'rb', newline='\t')
# can't have both text and binary
with pytest.raises(ValueError):
omnio.open('', 'rbt')
# only one compression type allowed
with pytest.raises(ValueError):
omnio.open('', 'rjz')
def test_rt_ascii():
path = 'tests/data/ascii.txt'
with omnio.open(path, 'rt', encoding='ascii') as fd:
data = fd.read()
assert type(data) is str
assert len(data) == 1392
def test_rbj():
with omnio.open('tests/data/flights-3m.csv.bz2', 'rbj') as fd:
data = fd.read()
assert type(data) is bytes
assert len(data) == 5535530 # uncompressed file size
def test_rt_utf8():
path = 'tests/data/utf-8.txt'
with omnio.open(path, 'rt', encoding='utf-8') as fd:
data = fd.read()
assert type(data) is str
assert len(data) == 874
def test_rt_iso8859():
path = 'tests/data/iso-8859-1.txt'
with omnio.open(path, 'rt', encoding='iso-8859-1') as fd:
data = fd.read()
assert type(data) is str
assert len(data) == 700
