def append_csv(self,
local_path,
encoding=None,
errors='strict',
**csvargs):
"""
Appends table to an existing CSV.
Additional additional key word arguments
are passed to ``csv.writer()``. So, e.g., to override the delimiter
from the default CSV dialect, provide the delimiter keyword argument.
`Args:`
local_path: str
The local path of an existing CSV file. If it ends in ".gz", the file will
be compressed.
encoding: str
The CSV encoding type for `csv.writer()
<https://docs.python.org/2/library/csv.html#csv.writer/>`_
errors: str
Raise an Error if encountered
\**csvargs: kwargs
``csv_writer`` optional arguments
`Returns:`
str
The path of the file
""" # noqa: W605
petl.appendcsv(self.table,
source=local_path,
encoding=encoding,
errors=errors,
**csvargs)
return local_path
def test_stringsource():
table1 = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2'))
# test writing to a string buffer
ss = StringSource()
etl.tocsv(table1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
if not PY2:
expect = expect.encode('ascii')
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
table2 = etl.fromcsv(StringSource(actual))
ieq(table1, table2)
ieq(table1, table2)
# test appending
etl.appendcsv(table1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
if not PY2:
expect = expect.encode('ascii')
eq_(expect, actual)
def export(data, output_file, source, csv_arg, errors, write_header, append):
"""Export the specified table of data to a csv file."""
existing_data = data.get(source)
if append is True:
petl.appendcsv(existing_data, output_file, errors=errors, **dict(csv_arg))
else:
petl.tocsv(existing_data, output_file, errors=errors, write_header=write_header, **dict(csv_arg))
def test_stringsource():
tbl1 = (('foo', 'bar'),
('a', '1'),
('b', '2'),
('c', '2'))
# test writing to a string buffer
ss = StringSource()
etl.tocsv(tbl1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
if not PY2:
expect = expect.encode('ascii')
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
tbl2 = etl.fromcsv(StringSource(actual))
ieq(tbl1, tbl2)
ieq(tbl1, tbl2)
# test appending
etl.appendcsv(tbl1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
if not PY2:
expect = expect.encode('ascii')
eq_(expect, actual)
def download_new_collection(cls) -> None:
# store small dictionary for later on transofrmation
planets_arr = {}
for planets in SWAPI.fetch_data(settings.SW_PLANETS_URL):
planets_arr.update({i['url']: i['name'] for i in planets})
create = True
file_name = '{}.csv'.format(time())
csv_path = Path(CSV_PATH, file_name)
for people in SWAPI.fetch_data(settings.SW_PEOPLE_URL):
table = etl.fromdicts(
people,
header=[
'name', 'height', 'mass', 'hair_color', 'skin_color',
'eye_color', 'birth_year', 'gender', 'homeworld', 'edited'
]).convert('edited', lambda v: v[0:10]).convert(
'homeworld',
lambda v: planets_arr.get(v, '')).rename('edited', 'date')
if create:
etl.tocsv(table, source=csv_path, write_header=True)
create = False
else:
etl.appendcsv(table, source=csv_path)
c = SWPeopleCollection()
c.file.name = file_name
c.save()
def test_tocsv_appendcsv_gz():
"""Test the tocsv and appendcsv function."""
# exercise function
table = (("foo", "bar"), ("a", 1), ("b", 2), ("c", 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + ".gz"
f.close()
tocsv(table, fn, delimiter="\t")
# check what it did
with gzip.open(fn, "rb") as o:
actual = csv.reader(o, delimiter="\t")
expect = [["foo", "bar"], ["a", "1"], ["b", "2"], ["c", "2"]]
ieq(expect, actual)
# check appending
table2 = (("foo", "bar"), ("d", 7), ("e", 9), ("f", 1))
appendcsv(table2, fn, delimiter="\t")
# check what it did
with gzip.open(fn, "rb") as o:
actual = csv.reader(o, delimiter="\t")
expect = [["foo", "bar"], ["a", "1"], ["b", "2"], ["c", "2"], ["d", "7"], ["e", "9"], ["f", "1"]]
ieq(expect, actual)
def test_tocsv_appendcsv_gz():
"""Test the tocsv and appendcsv function."""
# exercise function
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + '.gz'
f.close()
tocsv(table, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
ieq(expect, actual)
finally:
o.close()
# check appending
table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
appendcsv(table2, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
['d', '7'], ['e', '9'], ['f', '1']]
ieq(expect, actual)
finally:
o.close()
def save_to_csv_file(self, processed_person_array):
list_of_rows = [
self.get_person_row_for_csv(person_object)
for person_object in processed_person_array
]
etl.appendcsv(list_of_rows, self.csv_file_name)
util.log(
"info",
message="Saved to file",
context={"person_collection_id": self.person_collection.id},
)
def save_characters_to_file(generated_file_path, characters_pages):
etl.setheader(
etl.empty(),
settings.STAR_WARS_CHARACTERS_OUTPUT_FILE_HEADER_FIELDS,
).tocsv(generated_file_path)
logger.info('Created file: %s', generated_file_path)
for characters_page in characters_pages:
etl.appendcsv(
characters_page,
generated_file_path,
write_header=False,
)
logger.info('Added data to file: %s', generated_file_path)
def load(tables_by_id, output_folder, devices):
for device_id in tables_by_id:
name = valid_name(devices[device_id]['name'])
tbl_device_file = path.join(output_folder, f"{name}.csv")
if path.isfile(tbl_device_file):
tbl_old = petl.fromcsv(tbl_device_file, delimiter=';')
old_header = petl.header(tbl_old)
new_header = petl.header(tables_by_id[device_id])
if old_header == new_header:
petl.appendcsv(tables_by_id[device_id], source=tbl_device_file, delimiter=';')
else: # TODO: write to the new file
raise ValueError(f"Incompatible headers:\n old={old_header}\n new={new_header}")
else:
petl.tocsv(tables_by_id[device_id], tbl_device_file, delimiter=';')
def process_entity(name, processor):
page, url = 0, f'{settings.SWAPI_URL}{name}/'
while url:
response = session.get(url)
response.raise_for_status()
json = response.json()
table = processor(json['results'], page)
filename = str(folder / f'{name}.csv')
if page == 0:
etl.tocsv(table, filename)
else:
etl.appendcsv(table, filename)
url = json['next']
page += 1
def test_tocsv_appendcsv_gz():
"""Test the tocsv and appendcsv function."""
# exercise function
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
fn = f.name + '.gz'
f.close()
tocsv(table, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2']]
ieq(expect, actual)
finally:
o.close()
# check appending
table2 = (('foo', 'bar'),
('d', 7),
('e', 9),
('f', 1))
appendcsv(table2, fn, delimiter='\t')
# check what it did
o = gzip.open(fn, 'rb')
try:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2'],
['d', '7'],
['e', '9'],
['f', '1']]
ieq(expect, actual)
finally:
o.close()
def tocsv(data, target, append, **kwargs):
kwargs = filter_keys(
kwargs, ("encoding", "errors", "write_header", "dialect", "delimiter",
"quotechar", "escapechar", "doublequote", "skipinitialspace",
"lineterminator", "quoting"))
if append:
return etl.appendcsv(data, target, **kwargs)
else:
return etl.tocsv(data, target, **kwargs)
def fetch_collection(filepath):
addr = f'{settings.SWAPI_HOST}/api/people'
etl.tocsv([FIELDS_WE_NEED], filepath)
while addr:
response = requests.get(addr).json()
addr = response['next']
table_columns = [[item[column_name] for item in response['results']]
for column_name in FIELDS_WE_GET]
table = (etl.fromcolumns(table_columns, header=FIELDS_WE_GET).convert(
'homeworld', resolve_homeworld).addfield(
'date', lambda rec: rec['edited'].split('T')[0]).cutout(
'created').cutout('edited'))
etl.appendcsv(table, filepath)
return response['count']
def test_StringSource():
table1 = (('foo', 'bar'), ('a', '1'), ('b', '2'), ('c', '2'))
# test writing to a string buffer
ss = StringSource()
tocsv(table1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
table2 = fromcsv(StringSource(actual))
ieq(table1, table2)
ieq(table1, table2)
# test appending
appendcsv(table1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
eq_(expect, actual)
def test_tocsv_appendcsv():
"""Test the tocsv and appendcsv function."""
# exercise function
table = (('foo', 'bar'),
('a', 1),
('b', 2),
('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2']]
ieq(expect, actual)
# check appending
table2 = (('foo', 'bar'),
('d', 7),
('e', 9),
('f', 1))
appendcsv(table2, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'],
['a', '1'],
['b', '2'],
['c', '2'],
['d', '7'],
['e', '9'],
['f', '1']]
ieq(expect, actual)
def test_StringSource():
table1 = (('foo', 'bar'),
('a', '1'),
('b', '2'),
('c', '2'))
# test writing to a string buffer
ss = StringSource()
tocsv(table1, ss)
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\n"
actual = ss.getvalue()
eq_(expect, actual)
# test reading from a string buffer
table2 = fromcsv(StringSource(actual))
ieq(table1, table2)
ieq(table1, table2)
# test appending
appendcsv(table1, ss)
actual = ss.getvalue()
expect = "foo,bar\r\na,1\r\nb,2\r\nc,2\r\na,1\r\nb,2\r\nc,2\r\n"
eq_(expect, actual)
def test_tocsv_appendcsv():
"""Test the tocsv and appendcsv function."""
# exercise function
table = (('foo', 'bar'), ('a', 1), ('b', 2), ('c', 2))
f = NamedTemporaryFile(delete=False)
tocsv(table, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2']]
ieq(expect, actual)
# check appending
table2 = (('foo', 'bar'), ('d', 7), ('e', 9), ('f', 1))
appendcsv(table2, f.name, delimiter='\t')
# check what it did
with open(f.name, 'rb') as o:
actual = csv.reader(o, delimiter='\t')
expect = [['foo', 'bar'], ['a', '1'], ['b', '2'], ['c', '2'],
['d', '7'], ['e', '9'], ['f', '1']]
ieq(expect, actual)
import petl
import os
# Use this file to prepare CSV data from
# webrobot.io's kickstarter data
# change this listdir input to whatever
# your path to your data is
files = os.listdir(".")[0:-1]
print(files)
first = True
for i in files:
data = petl.fromcsv(i)
blurbs = petl.cut(data, 'blurb')
print(petl.look(blurbs))
if first:
petl.tocsv(blurbs, 'blurbs.csv')
first = False
else:
petl.appendcsv(blurbs, 'blurbs.csv')
# appendcsv
table = [['foo', 'bar'],
['d', 7],
['e', 42],
['f', 12]]
# look at an existing CSV file
from petl import look, fromcsv
testcsv = fromcsv('test.csv', delimiter='\t')
look(testcsv)
# append some data
look(table)
from petl import appendcsv
appendcsv(table, 'test.csv', delimiter='\t')
# look what it did
look(testcsv)
# topickle
table = [['foo', 'bar'],
['a', 1],
['b', 2],
['c', 2]]
from petl import topickle, look
look(table)
topickle(table, 'test.dat')
# look what it did
# appendcsv
table = [['foo', 'bar'],
['d', 7],
['e', 42],
['f', 12]]
# look at an existing CSV file
from petl import look, fromcsv
testcsv = fromcsv('test.csv', delimiter='\t')
look(testcsv)
# append some data
look(table)
from petl import appendcsv
appendcsv(table, 'test.csv', delimiter='\t')
# look what it did
look(testcsv)
# topickle
table = [['foo', 'bar'],
['a', 1],
['b', 2],
['c', 2]]
from petl import topickle, look
look(table)
topickle(table, 'test.dat')
# look what it did