def test_default_config(): config = omnio.default_config() for scheme in ["file", "http", "s3"]: assert isinstance(config[scheme], dict) omnio.open("tests/data/ascii.txt", 'rt', config=config)
def test_rtz(): with omnio.open('tests/data/flights-3m.csv.gz', 'rtz') as fd: reader = csv.reader(fd) headers = next(reader) assert headers == [ 'date', 'delay', 'distance', 'origin', 'destination' ] data = list(reader) assert len(data) == 231083 # number data rows
def test_read_chunk(): uri = 'http://example.com/example' data = os.urandom(1024) responses.add(responses.GET, uri, body=data, status=200) with omnio.open(uri, 'rb') as infile: chunk = infile.read(1000) assert len(chunk) == 1000 assert chunk == data[:1000] chunk = infile.read(1000) assert len(chunk) == 24 assert chunk == data[1000:]
def test_w(): path = 'tests/data/w.txt' try: with omnio.open(path, 'wt') as fd: fd.write('one\ntwo\nthree\n') except Exception: raise else: with open(path, 'r') as fd: assert list(fd) == ['one\n', 'two\n', 'three\n'] finally: os.remove(path)
def test_open_rt_utf8(): uri = 'http://example.com/example' data = _load_test_data('utf-8') responses.add( responses.GET, uri, body=data, status=200, content_type='text/plain; charset=UTF-8', ) with omnio.open(uri, 'rt') as infile: assert infile.read() == data.decode('utf-8')
def test_closed(): uri = 'http://example.com/example' data = b'foo bar baz' responses.add(responses.GET, uri, body=data, status=200) f = omnio.open(uri, 'rb') f.close() # none of these operations are allowed on a closed file with pytest.raises(ValueError): f.read() with pytest.raises(ValueError): next(f) with pytest.raises(ValueError): iter(f)
def test_read_connection_error(monkeypatch): class Client: def __init__(self, *args, **kwargs): pass def get_object(self, Bucket=None, Key=None): raise botocore.exceptions.EndpointConnectionError( endpoint_url="test/") monkeypatch.setattr("boto3.client", Client) with pytest.raises(ConnectionError): with omnio.open("s3://bucket/key"): pass
def test_wbj(): path = 'tests/data/wbj.txt.bz2' data = os.urandom(1024) try: with omnio.open(path, 'wbj') as fd: fd.write(data) except Exception: raise else: with open(path, 'rb') as fd: with bz2.open(fd) as gz: assert gz.read() == data finally: os.remove(path)
def test_wtz(): path = 'tests/data/wtz.txt.gz' data = 'unicode string to be seamlessly compressed' try: with omnio.open(path, 'wtz') as fd: fd.write(data) except Exception: raise else: with open(path, 'rb') as fd: with gzip.open(fd, 'rt') as gz: assert gz.read() == data finally: os.remove(path)
def test_iter(): uri = 'http://example.com/example' data = _load_test_data('utf-8') responses.add( responses.GET, uri, body=data, status=200, content_type='text/plain; charset=UTF-8', ) expect_first = ( 'Οὐχὶ ταὐτὰ παρίσταταί μοι γιγνώσκειν, ὦ ἄνδρες ' '᾿Αθηναῖοι,\n'.encode('utf-8') ) expect_last = 'τελευτῆς ὁντινοῦν ποιεῖσθαι λόγον.'.encode('utf-8') with omnio.open(uri, 'rb') as infile: lines = list(infile) assert len(lines) == 16 assert lines[0] == expect_first assert lines[-1] == expect_last
def test_invalid_mode(): with pytest.raises(ValueError): omnio.open('', 'q')
def test_invalid_combination(): # can't combine read and write with pytest.raises(ValueError): omnio.open('', 'rw') # can't specify encoding in binary mode with pytest.raises(ValueError): omnio.open('', 'rb', encoding='utf-8') # can't specify errors in binary mode with pytest.raises(ValueError): omnio.open('', 'rb', errors='strict') # can't specify newline in binary mode with pytest.raises(ValueError): omnio.open('', 'rb', newline='\t') # can't have both text and binary with pytest.raises(ValueError): omnio.open('', 'rbt') # only one compression type allowed with pytest.raises(ValueError): omnio.open('', 'rjz')
def test_rt_ascii(): path = 'tests/data/ascii.txt' with omnio.open(path, 'rt', encoding='ascii') as fd: data = fd.read() assert type(data) is str assert len(data) == 1392
def test_rbj(): with omnio.open('tests/data/flights-3m.csv.bz2', 'rbj') as fd: data = fd.read() assert type(data) is bytes assert len(data) == 5535530 # uncompressed file size
def test_rt_utf8(): path = 'tests/data/utf-8.txt' with omnio.open(path, 'rt', encoding='utf-8') as fd: data = fd.read() assert type(data) is str assert len(data) == 874
def test_rt_iso8859(): path = 'tests/data/iso-8859-1.txt' with omnio.open(path, 'rt', encoding='iso-8859-1') as fd: data = fd.read() assert type(data) is str assert len(data) == 700