def test_csv_with_several_str_field(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{
'field': f'"{idx}"'
} for idx in range(100)])
records = list(extract_data_from_csv(temp_csv_file.name))
assert sorted(records) == sorted(
[MetadataRecord('field', f'{idx}') for idx in range(100)])
def test_csv_gathering(monkeypatch, temp_csv_file, temp_db_file, capsys):
write_csv(temp_csv_file, ['field_one', 'field_two', 'field_three'], [
{'field_one': 10, 'field_two': '"abc"', 'field_three': 'null'},
{'field_one': 'null', 'field_two': 'null', 'field_three': 'null'},
{'field_one': 'null', 'field_two': '"def"', 'field_three': 0},
{'field_one': 90, 'field_two': '"jkl"', 'field_three': 120},
])
def namespace(_):
return argparse.Namespace(crawl=temp_csv_file.name, database_path=temp_db_file.name)
monkeypatch.setattr(argparse.ArgumentParser, "parse_args", namespace)
main()
def namespace(_):
return argparse.Namespace(crawl=None, describe=temp_csv_file.name, database_path=temp_db_file.name)
monkeypatch.setattr(argparse.ArgumentParser, "parse_args", namespace)
main()
captured = capsys.readouterr()
assert captured.out.split('\n') == [
f'File: {temp_csv_file.name}',
'Total entries: 3',
'Fields:',
'\tfield_one, Integer, 2, 2',
'\tfield_two, String, 3, 1',
'\tfield_three, Integer, 2, 2',
'',
]
def test_csv_missing_value_for_column(temp_csv_file):
write_csv(temp_csv_file, ['field_one', 'field_two'], [{
'field_one': '"abc"',
}])
with pytest.raises(ExtractionError) as exc:
list(extract_data_from_csv(temp_csv_file.name))
info = exc.value
assert info.args[0] == "Missing value for column 'field_two' at line 2"
def test_csv_with_multiple_fields_multiple_rows(temp_csv_file):
write_csv(temp_csv_file, ['field_one', 'field_two'], [{
'field_one': '"abc"',
'field_two': 50
} for _ in range(1000)])
records = list(extract_data_from_csv(temp_csv_file.name))
assert sorted(records) == sorted(
[MetadataRecord('field_one', 'abc')] * 1000 +
[MetadataRecord('field_two', 50)] * 1000, )
def test_csv_invalid_string_format(temp_csv_file):
write_csv(temp_csv_file, ['field_one'], [{
'field_one': 'abc',
}])
with pytest.raises(ExtractionError) as exc:
list(extract_data_from_csv(temp_csv_file.name))
info = exc.value
assert info.args[
0] == "Unknown type for value 'abc' (column 'field_one') at line 2"
def test_csv_with_multiple_fields(temp_csv_file):
write_csv(temp_csv_file, ['field_one', 'field_two', 'field_three'],
[{
'field_one': '"abc"',
'field_two': 50,
'field_three': 'null'
}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert sorted(records) == sorted([
MetadataRecord('field_one', 'abc'),
MetadataRecord('field_two', 50),
MetadataRecord('field_three', None),
])
def test_csv_with_several_nulls_field(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{
'field': 'null'
} for _ in range(100)])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('field', None)] * 100
def test_csv_with_single_null_field(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{'field': 'null'}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('field', None)]
def test_csv_with_single_str_field(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{'field': '"string_value"'}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('field', "string_value")]
def test_csv_with_several_int_fields(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{'field': idx} for idx in range(100)])
records = list(extract_data_from_csv(temp_csv_file.name))
assert sorted(records) == sorted(
[MetadataRecord('field', idx) for idx in range(100)])
def test_csv_unicode_value(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{'field': '"短消息"'}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('field', '短消息')]
def test_csv_with_single_int_field(temp_csv_file):
write_csv(temp_csv_file, ['field'], [{'field': 50}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('field', 50)]
def test_csv_unicode_column_name(temp_csv_file):
write_csv(temp_csv_file, ['短消息'], [{'短消息': 50}])
records = list(extract_data_from_csv(temp_csv_file.name))
assert records == [MetadataRecord('短消息', 50)]