def test_filter_csv(tmpdir):
infile = str(tmpdir.join('input.csv'))
outfile1 = str(tmpdir.join('output1.csv'))
outfile2 = str(tmpdir.join('output2.csv'))
with open(infile, 'w') as f:
f.write(SIMPLE_CSV)
code = """
InCSV({0!r}, fieldnames=['id', 'name', 'price']) | {{
filter(float(item.price) >= 90) | OutCSV({1!r}) ,
filter(float(item.price) < 20) | OutCSV({2!r})
}}
""".format(infile, outfile1, outfile2)
pipe = parser.parse(code)
execute(pipe)
assert open(outfile1).read() == EXP_CSV_OUT1
assert open(outfile2).read() == EXP_CSV_OUT2
def test_filter_csv(tmpdir):
infile = str(tmpdir.join('input.csv'))
outfile1 = str(tmpdir.join('output1.csv'))
outfile2 = str(tmpdir.join('output2.csv'))
with open(infile, 'w') as f:
f.write(SIMPLE_CSV)
code = """
InCSV({0!r}, fieldnames=['id', 'name', 'price']) | {{
filter(float(item.price) >= 90) | OutCSV({1!r}) ,
filter(float(item.price) < 20) | OutCSV({2!r})
}}
""".format(infile, outfile1, outfile2)
pipe = parser.parse(code)
execute(pipe)
assert open(outfile1).read() == EXP_CSV_OUT1
assert open(outfile2).read() == EXP_CSV_OUT2
def test_csv_to_json(tmpdir):
infile = str(tmpdir.join('input.csv'))
outfile = str(tmpdir.join('output.json'))
with open(infile, 'w') as f:
f.write("id,name,price\r\n")
f.write("1,item-1,10.0\r\n")
f.write("2,item-2,20.0\r\n")
f.write("3,item-3,30.0\r\n")
code = """
InCSV({0!r}, header=True)
| Map(item._asdict())
| List() | OutJSON({1!r})
""".format(infile, outfile)
pipe = parser.parse(code)
execute(pipe)
assert json.load(open(outfile)) == [
{'id': '1', 'name': 'item-1', 'price': '10.0'},
{'id': '2', 'name': 'item-2', 'price': '20.0'},
{'id': '3', 'name': 'item-3', 'price': '30.0'},
]
code = """
InCSV({0!r}, header=True)
| Map(item._asdict())
| Transform(id=int(item['id']), price=float(item['price']))
| List() | OutJSON({1!r})
""".format(infile, outfile)
pipe = parser.parse(code)
execute(pipe)
assert json.load(open(outfile)) == [
{'id': 1, 'name': 'item-1', 'price': 10.0},
{'id': 2, 'name': 'item-2', 'price': 20.0},
{'id': 3, 'name': 'item-3', 'price': 30.0},
]
def test_simple_pipeline_parsed():
pipe = parser.parse("""
IN() | filter(item.price >= 50) | OUT()
""")
result = execute(pipe)
assert len(result) == 6
assert result[0] == (5, 'item-5', 50.0)
assert result[1] == (6, 'item-6', 60.0)
assert result[2] == (7, 'item-7', 70.0)
assert result[3] == (8, 'item-8', 80.0)
assert result[4] == (9, 'item-9', 90.0)
assert result[5] == (10, 'item-10', 100.0)
def test_simple_pipeline():
D1 = Device.from_star('IN')
cond = Expression.from_string('item.price >= 50')
D2 = Device.from_star('filter', cond)
D3 = Device.from_star('OUT')
pipe = Pipeline([D1, D2, D3])
result = execute(pipe)
assert len(result) == 6
assert result[0] == (5, 'item-5', 50.0)
assert result[1] == (6, 'item-6', 60.0)
assert result[2] == (7, 'item-7', 70.0)
assert result[3] == (8, 'item-8', 80.0)
assert result[4] == (9, 'item-9', 90.0)
assert result[5] == (10, 'item-10', 100.0)
def test_simple_block():
pipe = parser.parse("""
IN() | {
filter(item.price >= 90) | OUT() ,
filter(item.price < 20) | OUT()
}
""")
result = execute(pipe)
assert isinstance(result, tuple)
assert len(result) == 2
assert result[0] == [
(9, 'item-9', 90.0),
(10, 'item-10', 100.0),
]
assert result[1] == [
(1, 'item-1', 10.0),
]
def test_csv_to_json(tmpdir):
infile = str(tmpdir.join('input.csv'))
outfile = str(tmpdir.join('output.json'))
with open(infile, 'w') as f:
f.write("id,name,price\r\n")
f.write("1,item-1,10.0\r\n")
f.write("2,item-2,20.0\r\n")
f.write("3,item-3,30.0\r\n")
code = """
InCSV({0!r}, header=True)
| Map(item._asdict())
| List() | OutJSON({1!r})
""".format(infile, outfile)
pipe = parser.parse(code)
execute(pipe)
assert json.load(open(outfile)) == [
{
'id': '1',
'name': 'item-1',
'price': '10.0'
},
{
'id': '2',
'name': 'item-2',
'price': '20.0'
},
{
'id': '3',
'name': 'item-3',
'price': '30.0'
},
]
code = """
InCSV({0!r}, header=True)
| Map(item._asdict())
| Transform(id=int(item['id']), price=float(item['price']))
| List() | OutJSON({1!r})
""".format(infile, outfile)
pipe = parser.parse(code)
execute(pipe)
assert json.load(open(outfile)) == [
{
'id': 1,
'name': 'item-1',
'price': 10.0
},
{
'id': 2,
'name': 'item-2',
'price': 20.0
},
{
'id': 3,
'name': 'item-3',
'price': 30.0
},
]
