def test_step_cell_fill_direction_down():
source = Resource(path="data/transform.csv")
source.infer()
target = transform(
source,
steps=[
steps.cell_replace(pattern="france", replace=None),
steps.cell_fill(direction="down"),
],
)
assert target.schema == source.schema
assert target.read_rows() == [
{
"id": 1,
"name": "germany",
"population": 83
},
{
"id": 2,
"name": "germany",
"population": 66
},
{
"id": 3,
"name": "spain",
"population": 47
},
]
def test_step_cell_replace_using_regex():
source = Resource(path="data/transform.csv")
source.infer()
target = transform(
source,
steps=[
steps.cell_replace(pattern="<regex>.*r.*",
replace="center",
field_name="name"),
],
)
assert target.schema == source.schema
assert target.read_rows() == [
{
"id": 1,
"name": "center",
"population": 83
},
{
"id": 2,
"name": "center",
"population": 66
},
{
"id": 3,
"name": "spain",
"population": 47
},
]
def test_step_cell_replace_with_field_name():
source = Resource(path="data/transform.csv")
source.infer()
target = transform(
source,
steps=[
steps.cell_replace(pattern="france",
replace="FRANCE",
field_name="id"),
],
)
assert target.schema == source.schema
assert target.read_rows() == [
{
"id": 1,
"name": "germany",
"population": 83
},
{
"id": 2,
"name": "france",
"population": 66
},
{
"id": 3,
"name": "spain",
"population": 47
},
]
def test_step_cell_fill_direction_left():
source = Resource(path="data/transform.csv")
source.infer()
source.schema.get_field("id").type = "string"
source.schema.get_field("population").type = "string"
target = transform(
source,
steps=[
steps.cell_replace(pattern="france", replace=None),
steps.cell_fill(direction="left"),
],
)
assert target.schema == source.schema
assert target.read_rows() == [
{
"id": "1",
"name": "germany",
"population": "83"
},
{
"id": "2",
"name": "66",
"population": "66"
},
{
"id": "3",
"name": "spain",
"population": "47"
},
]
def test_step_cell_fill_direction_right():
source = Resource(path="data/transform.csv")
target = transform(
source,
steps=[
steps.field_update(name="id", type="string"),
steps.field_update(name="population", type="string"),
steps.cell_replace(pattern="france", replace=None),
steps.cell_fill(direction="right"),
],
)
print(target.read_rows())
assert target.schema == {
"fields": [
{
"name": "id",
"type": "string"
},
{
"name": "name",
"type": "string"
},
{
"name": "population",
"type": "string"
},
]
}
assert target.read_rows() == [
{
"id": "1",
"name": "germany",
"population": "83"
},
{
"id": "2",
"name": "2",
"population": "66"
},
{
"id": "3",
"name": "spain",
"population": "47"
},
]
def test_step_cell_fill():
source = Resource(path="data/transform.csv")
target = transform(
source,
steps=[
steps.cell_replace(pattern="france", replace=None),
steps.cell_fill(field_name="name", value="FRANCE"),
],
)
assert target.schema == {
"fields": [
{
"name": "id",
"type": "integer"
},
{
"name": "name",
"type": "string"
},
{
"name": "population",
"type": "integer"
},
]
}
assert target.read_rows() == [
{
"id": 1,
"name": "germany",
"population": 83
},
{
"id": 2,
"name": "FRANCE",
"population": 66
},
{
"id": 3,
"name": "spain",
"population": 47
},
]
