def test_tocsv_appendcsv_gz():
# exercise function
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + '.gz'
f.close()
tocsv(table, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
ieq(expect, actual)
finally:
o.close()
# check appending
table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
appendcsv(table2, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
['d', '7'], ['e', '9'], ['f', '1']]
ieq(expect, actual)
finally:
o.close()
def _write_read_file_into_url(base_url, filename, compression=None, pkg=None):
if not _is_installed(pkg, filename):
return
source_url = _build_source_url_from(base_url, filename, compression)
if source_url is None:
return
actual = None
if ".avro" in filename:
toavro(_table, source_url)
actual = fromavro(source_url)
elif ".xlsx" in filename:
toxlsx(_table, source_url, 'test1', mode='overwrite')
toxlsx(_table2, source_url, 'test2', mode='add')
actual = fromxlsx(source_url, 'test1')
elif ".xls" in filename:
toxls(_table, source_url, 'test')
actual = fromxls(source_url, 'test')
elif ".json" in filename:
tojson(_table, source_url)
actual = fromjson(source_url)
elif ".csv" in filename:
tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
actual = fromcsv(source_url, encoding="ascii")
if actual is not None:
_show__rows_from("Expected:", _table)
_show__rows_from("Actual:", actual)
ieq(_table, actual)
ieq(_table, actual) # verify can iterate twice
else:
print("\n - %s SKIPPED " % filename, file=sys.stderr, end="")
def _write_read_file_into_url(base_url, filename, compression=None):
if ".avro" in filename and not _has_avro:
return
is_local = base_url.startswith("./")
if compression is not None:
if is_local:
return
filename = filename + "." + compression
codec = fsspec.utils.infer_compression(filename)
if codec is None:
print("\n - %s SKIPPED " % filename, file=sys.stderr, end="")
return
print("\n - %s " % filename, file=sys.stderr, end="")
if is_local:
source_url = base_url + filename
else:
source_url = os.path.join(base_url, filename)
_show__rows_from("Expected:", _table)
if ".avro" in filename:
toavro(_table, source_url)
actual = fromavro(source_url)
else:
tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
actual = fromcsv(source_url, encoding="ascii")
_show__rows_from("Actual:", actual)
ieq(_table, actual)
ieq(_table, actual) # verify can iterate twice
def test_totsv_appendtsv():
# exercise function
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
totsv(table, f.name)
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
ieq(expect, actual)
# check appending
table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
appendtsv(table2, f.name)
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
['d', '7'], ['e', '9'], ['f', '1']]
ieq(expect, actual)
# check explicit no header
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, delimiter='\t', write_header=False)
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['a', '1'], ['b', '2'], ['c', '2']]
ieq(expect, actual)
def test_tocsv():
tbl = ((u'name', u'id'), (u'Արամ Խաչատրյան', 1), (u'Johann Strauß', 2),
(u'Вагиф Сәмәдоғлу', 3), (u'章子怡', 4))
fn = NamedTemporaryFile().name
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')
expect = (u"name,id\n"
u"Արամ Խաչատրյան,1\n"
u"Johann Strauß,2\n"
u"Вагиф Сәмәдоғлу,3\n"
u"章子怡,4\n")
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)
# Test with write_header=False
tbl = ((u'name', u'id'), (u'Արամ Խաչատրյան', 1), (u'Johann Strauß', 2),
(u'Вагиф Сәмәдоғлу', 3), (u'章子怡', 4))
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n', write_header=False)
expect = (u"Արամ Խաչատրյան,1\n"
u"Johann Strauß,2\n"
u"Вагиф Сәмәдоғлу,3\n"
u"章子怡,4\n")
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)
def test_tocsv_appendcsv():
# exercise function
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
f.close()
tocsv(table, f.name, encoding='ascii', lineterminator='\n')
# check what it did
with open(f.name, 'rb') as o:
data = [b'foo,bar', b'a,1', b'b,2', b'c,2']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
# check appending
table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
appendcsv(table2, f.name, encoding='ascii', lineterminator='\n')
# check what it did
with open(f.name, 'rb') as o:
data = [b'foo,bar', b'a,1', b'b,2', b'c,2', b'd,7', b'e,9', b'f,1']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
def fetch(request):
people = fetch_people_table()
uid = uuid.uuid4()
fname = '{0}.csv'.format(uid)
full_fname = os.path.join(settings.CSV_DIR, fname)
tocsv(people, full_fname)
CSVDownload.objects.create(uuid=uid)
return redirect(to=reverse('csv_list'))
def test_tocsv_none():
tbl = ((u'col1', u'colNone'), (u'a', 1), (u'b', None), (u'c', None), (u'd',
4))
fn = NamedTemporaryFile().name
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')
expect = (u'col1,colNone\n' u'a,1\n' u'b,\n' u'c,\n' u'd,4\n')
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)
def test_tocsv_appendcsv():
# exercise function
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2']]
ieq(expect, actual)
# check appending
table2 = (('foo', 'bar'),
('d', 7),
('e', 9),
('f', 1))
appendcsv(table2, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2'],
['d', '7'],
['e', '9'],
['f', '1']]
ieq(expect, actual)
# check explicit no header
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, delimiter='\t', write_header=False)
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['a', '1'],
['b', '2'],
['c', '2']]
ieq(expect, actual)
def test_tocsv_appendcsv_gz():
# exercise function
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + '.gz'
f.close()
tocsv(table, fn, encoding='ascii', lineterminator='\n')
# check what it did
o = gzip.open(fn, 'rb')
try:
data = [b'foo,bar',
b'a,1',
b'b,2',
b'c,2']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
finally:
o.close()
# check appending
table2 = (('foo', 'bar'),
('d', 7),
('e', 9),
('f', 1))
appendcsv(table2, fn, encoding='ascii', lineterminator='\n')
# check what it did
o = gzip.open(fn, 'rb')
try:
data = [b'foo,bar',
b'a,1',
b'b,2',
b'c,2',
b'd,7',
b'e,9',
b'f,1']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
finally:
o.close()
def _write_read_file_into_url(base_url, filename, compression=None, pkg=None):
if pkg is not None:
if not _is_installed(pkg):
print("\n - %s SKIPPED " % filename, file=sys.stderr, end="")
return
is_local = base_url.startswith("./")
if compression is not None:
if is_local:
return
filename = filename + "." + compression
codec = fsspec.utils.infer_compression(filename)
if codec is None:
print("\n - %s SKIPPED " % filename, file=sys.stderr, end="")
return
print("\n - %s " % filename, file=sys.stderr, end="")
if is_local:
source_url = base_url + filename
else:
source_url = os.path.join(base_url, filename)
actual = None
if ".avro" in filename:
toavro(_table, source_url)
actual = fromavro(source_url)
elif ".xlsx" in filename:
toxlsx(_table, source_url, 'test1', mode='overwrite')
toxlsx(_table2, source_url, 'test2', mode='add')
actual = fromxlsx(source_url, 'test1')
elif ".xls" in filename:
toxls(_table, source_url, 'test')
actual = fromxls(source_url, 'test')
elif ".json" in filename:
tojson(_table, source_url)
actual = fromjson(source_url)
elif ".csv" in filename:
tocsv(_table, source_url, encoding="ascii", lineterminator="\n")
actual = fromcsv(source_url, encoding="ascii")
if actual is not None:
_show__rows_from("Expected:", _table)
_show__rows_from("Actual:", actual)
ieq(_table, actual)
ieq(_table, actual) # verify can iterate twice
else:
print("\n - %s SKIPPED " % filename, file=sys.stderr, end="")
def test_tocsv_appendcsv_gz():
# exercise function
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + '.gz'
f.close()
tocsv(table, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2']]
ieq(expect, actual)
finally:
o.close()
# check appending
table2 = (('foo', 'bar'),
('d', 7),
('e', 9),
('f', 1))
appendcsv(table2, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2'],
['d', '7'],
['e', '9'],
['f', '1']]
ieq(expect, actual)
finally:
o.close()
def test_tocsv_noheader():
# check explicit no header
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table,
f.name,
encoding='ascii',
lineterminator='\n',
write_header=False)
# check what it did
with open(f.name, 'rb') as o:
data = [b'a,1', b'b,2', b'c,2']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
def test_tocsv_noheader():
# check explicit no header
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, encoding='ascii', lineterminator='\n',
write_header=False)
# check what it did
with open(f.name, 'rb') as o:
data = [b'a,1',
b'b,2',
b'c,2']
# don't forget final terminator
expect = b'\n'.join(data) + b'\n'
actual = o.read()
eq_(expect, actual)
def test_tocsv_none():
tbl = ((u'col1', u'colNone'),
(u'a', 1),
(u'b', None),
(u'c', None),
(u'd', 4))
fn = NamedTemporaryFile().name
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')
expect = (
u'col1,colNone\n'
u'a,1\n'
u'b,\n'
u'c,\n'
u'd,4\n'
)
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)
def test_stringsource():
table1 = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2'))
# test writing to a string buffer
ss = StringSource()
tocsv(table1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
table2 = fromcsv(StringSource(actual))
ieq(table1, table2)
ieq(table1, table2)
# test appending
appendcsv(table1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
eq_(expect, actual)
def test_stringsource():
table1 = (("foo", "bar"), ("a", "1"), ("b", "2"), ("c", "2"))
# test writing to a string buffer
ss = StringSource()
tocsv(table1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
table2 = fromcsv(StringSource(actual))
ieq(table1, table2)
ieq(table1, table2)
# test appending
appendcsv(table1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
eq_(expect, actual)
def test_tocsv():
tbl = ((u'name', u'id'),
(u'Արամ Խաչատրյան', 1),
(u'Johann Strauß', 2),
(u'Вагиф Сәмәдоғлу', 3),
(u'章子怡', 4))
fn = NamedTemporaryFile().name
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n')
expect = (
u"name,id\n"
u"Արամ Խաչատրյան,1\n"
u"Johann Strauß,2\n"
u"Вагиф Сәмәдоғлу,3\n"
u"章子怡,4\n"
)
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)
# Test with write_header=False
tbl = ((u'name', u'id'),
(u'Արամ Խաչատրյան', 1),
(u'Johann Strauß', 2),
(u'Вагиф Сәмәдоғлу', 3),
(u'章子怡', 4))
tocsv(tbl, fn, encoding='utf-8', lineterminator='\n', write_header=False)
expect = (
u"Արամ Խաչատրյան,1\n"
u"Johann Strauß,2\n"
u"Вагиф Сәмәдоғлу,3\n"
u"章子怡,4\n"
)
uf = io.open(fn, encoding='utf-8', mode='rt', newline='')
actual = uf.read()
eq_(expect, actual)