コード例 #1
0
ファイル: test_pipes_execution.py プロジェクト: rshk/dq
def test_filter_csv(tmpdir):
    infile = str(tmpdir.join('input.csv'))
    outfile1 = str(tmpdir.join('output1.csv'))
    outfile2 = str(tmpdir.join('output2.csv'))

    with open(infile, 'w') as f:
        f.write(SIMPLE_CSV)

    code = """
    InCSV({0!r}, fieldnames=['id', 'name', 'price']) | {{
        filter(float(item.price) >= 90) | OutCSV({1!r}) ,
        filter(float(item.price) < 20) | OutCSV({2!r})
    }}
    """.format(infile, outfile1, outfile2)

    pipe = parser.parse(code)
    execute(pipe)

    assert open(outfile1).read() == EXP_CSV_OUT1
    assert open(outfile2).read() == EXP_CSV_OUT2
コード例 #2
0
ファイル: test_pipes_execution.py プロジェクト: rshk/dq
def test_filter_csv(tmpdir):
    infile = str(tmpdir.join('input.csv'))
    outfile1 = str(tmpdir.join('output1.csv'))
    outfile2 = str(tmpdir.join('output2.csv'))

    with open(infile, 'w') as f:
        f.write(SIMPLE_CSV)

    code = """
    InCSV({0!r}, fieldnames=['id', 'name', 'price']) | {{
        filter(float(item.price) >= 90) | OutCSV({1!r}) ,
        filter(float(item.price) < 20) | OutCSV({2!r})
    }}
    """.format(infile, outfile1, outfile2)

    pipe = parser.parse(code)
    execute(pipe)

    assert open(outfile1).read() == EXP_CSV_OUT1
    assert open(outfile2).read() == EXP_CSV_OUT2
コード例 #3
0
ファイル: test_pipes_execution.py プロジェクト: rshk/dq
def test_csv_to_json(tmpdir):
    infile = str(tmpdir.join('input.csv'))
    outfile = str(tmpdir.join('output.json'))

    with open(infile, 'w') as f:
        f.write("id,name,price\r\n")
        f.write("1,item-1,10.0\r\n")
        f.write("2,item-2,20.0\r\n")
        f.write("3,item-3,30.0\r\n")

    code = """
    InCSV({0!r}, header=True)
    | Map(item._asdict())
    | List() | OutJSON({1!r})
    """.format(infile, outfile)

    pipe = parser.parse(code)
    execute(pipe)

    assert json.load(open(outfile)) == [
        {'id': '1', 'name': 'item-1', 'price': '10.0'},
        {'id': '2', 'name': 'item-2', 'price': '20.0'},
        {'id': '3', 'name': 'item-3', 'price': '30.0'},
    ]

    code = """
    InCSV({0!r}, header=True)
    | Map(item._asdict())
    | Transform(id=int(item['id']), price=float(item['price']))
    | List() | OutJSON({1!r})
    """.format(infile, outfile)

    pipe = parser.parse(code)
    execute(pipe)

    assert json.load(open(outfile)) == [
        {'id': 1, 'name': 'item-1', 'price': 10.0},
        {'id': 2, 'name': 'item-2', 'price': 20.0},
        {'id': 3, 'name': 'item-3', 'price': 30.0},
    ]
コード例 #4
0
def test_simple_pipeline_parsed():
    pipe = parser.parse("""
    IN() | filter(item.price >= 50) | OUT()
    """)

    result = execute(pipe)

    assert len(result) == 6
    assert result[0] == (5, 'item-5', 50.0)
    assert result[1] == (6, 'item-6', 60.0)
    assert result[2] == (7, 'item-7', 70.0)
    assert result[3] == (8, 'item-8', 80.0)
    assert result[4] == (9, 'item-9', 90.0)
    assert result[5] == (10, 'item-10', 100.0)
コード例 #5
0
def test_simple_pipeline():
    D1 = Device.from_star('IN')
    cond = Expression.from_string('item.price >= 50')
    D2 = Device.from_star('filter', cond)
    D3 = Device.from_star('OUT')

    pipe = Pipeline([D1, D2, D3])
    result = execute(pipe)

    assert len(result) == 6
    assert result[0] == (5, 'item-5', 50.0)
    assert result[1] == (6, 'item-6', 60.0)
    assert result[2] == (7, 'item-7', 70.0)
    assert result[3] == (8, 'item-8', 80.0)
    assert result[4] == (9, 'item-9', 90.0)
    assert result[5] == (10, 'item-10', 100.0)
コード例 #6
0
def test_simple_block():
    pipe = parser.parse("""
    IN() | {
        filter(item.price >= 90) | OUT() ,
        filter(item.price < 20) | OUT()
    }
    """)
    result = execute(pipe)

    assert isinstance(result, tuple)
    assert len(result) == 2

    assert result[0] == [
        (9, 'item-9', 90.0),
        (10, 'item-10', 100.0),
    ]
    assert result[1] == [
        (1, 'item-1', 10.0),
    ]
コード例 #7
0
ファイル: test_pipes_execution.py プロジェクト: rshk/dq
def test_csv_to_json(tmpdir):
    infile = str(tmpdir.join('input.csv'))
    outfile = str(tmpdir.join('output.json'))

    with open(infile, 'w') as f:
        f.write("id,name,price\r\n")
        f.write("1,item-1,10.0\r\n")
        f.write("2,item-2,20.0\r\n")
        f.write("3,item-3,30.0\r\n")

    code = """
    InCSV({0!r}, header=True)
    | Map(item._asdict())
    | List() | OutJSON({1!r})
    """.format(infile, outfile)

    pipe = parser.parse(code)
    execute(pipe)

    assert json.load(open(outfile)) == [
        {
            'id': '1',
            'name': 'item-1',
            'price': '10.0'
        },
        {
            'id': '2',
            'name': 'item-2',
            'price': '20.0'
        },
        {
            'id': '3',
            'name': 'item-3',
            'price': '30.0'
        },
    ]

    code = """
    InCSV({0!r}, header=True)
    | Map(item._asdict())
    | Transform(id=int(item['id']), price=float(item['price']))
    | List() | OutJSON({1!r})
    """.format(infile, outfile)

    pipe = parser.parse(code)
    execute(pipe)

    assert json.load(open(outfile)) == [
        {
            'id': 1,
            'name': 'item-1',
            'price': 10.0
        },
        {
            'id': 2,
            'name': 'item-2',
            'price': 20.0
        },
        {
            'id': 3,
            'name': 'item-3',
            'price': 30.0
        },
    ]