def test_required_column_empty_fails_even_with_default(self):
"""Should fail if the column is required and the value is empty."""
iphone_data = {'title': 'iPhone 5C', 'price': ''}
ipad_data = {'title': 'iPad mini', 'price': '799'}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True,
'default': 999
},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
def test_required_column_with_data_is_transformed(self):
"""Should apply the transformation to the value"""
iphone_data = {"title": "iPhone 5C", "price": "799"}
ipad_data = {"title": "iPad mini", "price": "699"}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(
StringIO(csv_data),
columns=[
{"name": "title", "required": True},
{"name": "price", "required": True, "transform": lambda x: Decimal(x)},
],
)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {"title": "iPhone 5C", "price": Decimal("799")})
self.assertModelsEquals(ipad, {"title": "iPad mini", "price": Decimal("699")})
def test_default_value_is_not_transformed(self):
"""Shouldn't apply no transformation if the value is missing and the default value is being used"""
iphone_data = {"title": "iPhone 5C", "price": ""}
ipad_data = {"title": "iPad mini", "price": ""}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)
)
mocked_validator = mock.MagicMock(return_value=True)
reader = smartcsv.reader(
StringIO(csv_data),
columns=[
{"name": "title", "required": True},
{"name": "price", "required": False, "transform": mocked_validator, "default": 899},
],
)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {"title": "iPhone 5C", "price": 899})
self.assertModelsEquals(ipad, {"title": "iPad mini", "price": 899})
self.assertEqual(mocked_validator.call_count, 0)
def test_error_exception_is_reported_without_fail_fast(self):
"""reader.errors should contain the exception that happenend with the value transformation"""
iphone_data = {"title": "iPhone 5C", "price": "INVALID"}
ipad_data = {"title": "iPad mini", "price": "699"}
iphone_row = "{title},{price}".format(**iphone_data)
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row=iphone_row, ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(
StringIO(csv_data),
columns=[
{"name": "title", "required": True},
{"name": "price", "required": True, "transform": lambda x: Decimal(x)},
],
fail_fast=False,
)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(ipad, dict))
self.assertModelsEquals(ipad, {"title": "iPad mini", "price": Decimal("699")})
self.assertTrue(reader.errors is not None)
self.assertTrue("rows" in reader.errors)
self.assertEqual(len(reader.errors["rows"]), 1) # 1 row failing
self.assertRowError(reader.errors, iphone_row, 0, "transform")
def test_not_required_value_is_provided(self):
"""Shouldn't fail and should have an the correct value"""
iphone_data = {
'title': 'iPhone 5C',
'price': '799'
}
ipad_data = {
'title': 'iPad mini',
'price': '699'
}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': False},
{'name': 'price', 'required': False},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_required_value_is_missing_and_dont_fail_fast(self):
"""Shouldn't fail fast, instead report errors on reader.errors"""
iphone_data = {
'title': 'iPhone 5C'
}
ipad_data = {
'title': 'iPad mini',
'price': '699'
}
iphone_row = "{title},".format(**iphone_data)
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row=iphone_row,
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{'name': 'price', 'required': True},
], fail_fast=False)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(ipad, dict))
self.assertModelsEquals(ipad, ipad_data)
self.assertTrue(reader.errors is not None)
self.assertTrue('rows' in reader.errors)
self.assertEqual(len(reader.errors['rows']), 1) # 1 row failing
self.assertRowError(
reader.errors, iphone_row, 0, 'price')
def test_not_required_column_empty_returns_default_value(self):
"""Should return the default value if no other is provided"""
iphone_data = {'title': 'iPhone 5C', 'price': ''}
ipad_data = {'title': 'iPad mini', 'price': '799'}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': False,
'default': 999
},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {'title': 'iPhone 5C', 'price': 999})
self.assertModelsEquals(ipad, ipad_data)
def test_valid_csv_with_blank_lines(self):
"""Should be valid even if there are blank lines"""
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA),
ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_invalid_value_is_passed_and_exception_is_raised(self):
"""Should not validate and raise a exception (fail_fast=True)"""
iphone_data = {
'title': 'iPhone 5C',
'price': 'INVALID'
}
ipad_data = {
'title': 'iPad mini',
'price': '699'
}
csv_data = """
title,price
{iphone_row}
{ipad_row}""".format(
iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{
'name': 'price',
'required': True,
'validator': is_number
},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
def test_valid_csv_with_blank_lines(self):
"""Should be valid even if there are blank lines"""
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA),
ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError('Usage: importproducts ' + self.args)
filepath = args[0]
print 'Importing persons from {0}...'.format(filepath)
stats = {'imported': 0, 'skipped': 0}
with open(filepath) as f:
reader = smartcsv.reader(f, columns=COLUMNS, fail_fast=False)
for person in reader:
try:
Profile.objects.create(
email=person['email'],
lg_full_name=person['name'],
lg_timezone=person['timezone']
)
except IntegrityError:
stats['skipped'] += 1
else:
stats['imported'] += 1
print ('Finished with stats: '
'imported {imported}, skipped {skipped}').format(**stats)
def test_choice_field_is_required_and_missing_value_is_invalid(self):
"""Should fail to validate if a required choice field is missing"""
csv_data = """
title,currency,price
iPhone 5c blue,USD,799
iPad mini,,699
"""
columns = [{
'name': 'title',
'required': True
}, {
'name': 'currency',
'required': True,
'choices': CURRENCY_CHOICES
}, {
'name': 'price',
'required': True,
'validator': is_number
}]
reader = smartcsv.reader(StringIO(csv_data), columns=columns)
iphone = next(reader)
self.assertRaises(InvalidCSVException, lambda: next(reader))
self.assertModelsEquals(iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': '799'
})
def test_valid_csv_with_some_fields_not_required_empty(self):
"""Should be valid if some not required fields are filled and others don't"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['subcategory'] = ""
iphone_data['image_url'] = ""
ipad_data['image_url'] = ""
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertEqual(iphone['title'], 'iPhone 5c blue')
self.assertEqual(ipad['title'], 'iPad mini')
self.assertEqual(iphone['subcategory'], '')
self.assertEqual(ipad['subcategory'], 'Apple')
def test_valid_csv_with_columns_names_with_whitespaces(self):
"""Should be valid if all data is passed"""
csv_data = """
title,category,subcategory,currency,price,url,Image URL
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA),
ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA),
)
columns = COLUMNS_1[:]
image_column = columns[-1].copy()
image_column['name'] = "Image URL"
columns = columns[:-1] + [image_column]
reader = smartcsv.reader(StringIO(csv_data), columns=columns)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertEqual(iphone['Image URL'], 'http://apple.com/iphone.jpg')
self.assertEqual(ipad['Image URL'], 'http://apple.com/ipad.jpg')
def test_choice_field_is_required_and_missing_value_is_invalid(self):
"""Should fail to validate if a required choice field is missing"""
csv_data = """
title,currency,price
iPhone 5c blue,USD,799
iPad mini,,699
"""
columns = [
{'name': 'title', 'required': True},
{
'name': 'currency',
'required': True,
'choices': CURRENCY_CHOICES
},
{
'name': 'price',
'required': True,
'validator': is_number
}
]
reader = smartcsv.reader(StringIO(csv_data), columns=columns)
iphone = next(reader)
self.assertRaises(InvalidCSVException, lambda: next(reader))
self.assertModelsEquals(iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': '799'
})
def test_skip_column_doesnt_invoke_validator(self):
"""Should return the object without the skipped column"""
iphone_data = {'title': 'iPhone 5C', 'price': '799'}
ipad_data = {'title': 'iPad mini', 'price': '699'}
csv_data = """
title,price
{iphone_row}
{ipad_row}""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
mocked_validator = mock.MagicMock(return_value=True)
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'validator': mocked_validator,
'skip': True
},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {'title': 'iPhone 5C'})
self.assertModelsEquals(ipad, {'title': 'iPad mini'})
self.assertEqual(mocked_validator.call_count, 0)
def test_valid_csv_with_some_fields_not_required_empty(self):
"""Should be valid if some not required fields are filled and others don't"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['subcategory'] = ""
iphone_data['image_url'] = ""
ipad_data['image_url'] = ""
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertEqual(iphone['title'], 'iPhone 5c blue')
self.assertEqual(ipad['title'], 'iPad mini')
self.assertEqual(iphone['subcategory'], '')
self.assertEqual(ipad['subcategory'], 'Apple')
def test_column_validator_is_not_callable(self):
"""Should fail a validator is not callable"""
columns = [{"name": "title", "required": True}, {"name": "price", "required": True, "validator": "8"}]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_column_doesnt_have_a_name(self):
"""Should fail if a column doesn't have a name"""
columns = [{"required": True}, {"name": "price", "required": True}]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_default_value_is_empty(self):
"""Should fail a default value is empty"""
columns = [{"name": "title", "required": True}, {"name": "price", "required": True, "default": ""}]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_valid_and_value_transformed_with_only_required_data(self):
"""Should transform all values with only required data present"""
csv_data = """
title,currency,price,in_stock
iPhone 5c blue,USD,799,
iPad mini,USD,699,
"""
reader = smartcsv.reader(StringIO(csv_data),
columns=COLUMNS_WITH_VALUE_TRANSFORMATIONS)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(
iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': Decimal('799'),
'in_stock': ''
})
self.assertModelsEquals(
ipad, {
'title': 'iPad mini',
'currency': 'USD',
'price': Decimal('699'),
'in_stock': ''
})
def test_column_with_missing_data_is_not_transformed(self):
"""Shouldn't invoke the transform function if no value is passed"""
iphone_data = {'title': 'iPhone 5C', 'price': ''}
ipad_data = {'title': 'iPad mini', 'price': ''}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
mocked_validator = mock.MagicMock(return_value=True)
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': False,
'transform': mocked_validator
},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
self.assertEqual(mocked_validator.call_count, 0)
def test_valid_choice_is_provided(self):
"""Should not fail and have the value of the selected choice"""
iphone_data = {
'title': 'iPhone 5C',
'currency': 'USD'
}
ipad_data = {
'title': 'iPad mini',
'currency': 'ARS'
}
csv_data = """
title,currency
{iphone_row}
{ipad_row}""".format(
iphone_row="{title},{currency}".format(**iphone_data),
ipad_row="{title},{currency}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{
'name': 'currency',
'required': True,
'choices': CURRENCY_CHOICES
}
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_not_required_column_empty_returns_default_value(self):
"""Should return the default value if no other is provided"""
iphone_data = {
'title': 'iPhone 5C',
'price': ''
}
ipad_data = {
'title': 'iPad mini',
'price': '799'
}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{'name': 'price', 'required': False, 'default': 999},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {
'title': 'iPhone 5C',
'price': 999
})
self.assertModelsEquals(ipad, ipad_data)
def test_valid_csv_with_different_choices(self):
"""All choices for a field should be valid"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['subcategory'] = ""
iphone_data['image_url'] = ""
ipad_data['subcategory'] = ""
ipad_data['image_url'] = ""
ipad_data['currency'] = "ARS"
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_required_column_empty_fails_even_with_default(self):
"""Should fail if the column is required and the value is empty."""
iphone_data = {
'title': 'iPhone 5C',
'price': ''
}
ipad_data = {
'title': 'iPad mini',
'price': '799'
}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{'name': 'price', 'required': True, 'default': 999},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
def test_valid_csv_with_a_custom_dialect(self):
"""Should be valid if all data is passed"""
piped_template = VALID_TEMPLATE_STR.replace(',', '|')
csv_data = """
title|category|subcategory|currency|price|url|image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=piped_template.format(**IPHONE_DATA),
ipad_data=piped_template.format(**IPAD_DATA),
)
csv.register_dialect('pipes', delimiter='|')
reader = smartcsv.reader(StringIO(csv_data),
dialect='pipes',
columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_valid_data_and_skip_lines_with_header(self):
"""Should skip the first N lines and parse data ok with header"""
csv_data = """
Generated by Autobot 2000 - V0.1.2
----------
This next is intentionally left blank
-- Beginning of content
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA),
ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA),
)
reader = smartcsv.reader(
StringIO(csv_data), columns=COLUMNS_1, skip_lines=6)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_required_value_is_missing_and_fail_fast(self):
"""Should fail fast and report the error"""
iphone_data = {
'title': 'iPhone 5C'
}
ipad_data = {
'title': 'iPad mini',
'price': '699'
}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(
iphone_row="{title},".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{'name': 'price', 'required': True},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
else:
assert False, "Shouldn't reach this state"
def test_invalid_value_is_passed_and_exception_is_raised(self):
"""Should not validate and raise a exception (fail_fast=True)"""
iphone_data = {'title': 'iPhone 5C', 'price': 'INVALID'}
ipad_data = {'title': 'iPad mini', 'price': '699'}
csv_data = """
title,price
{iphone_row}
{ipad_row}""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True,
'validator': is_number
},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
def test_valid_and_value_transformed_with_only_required_data(self):
"""Should transform all values with only required data present"""
csv_data = """
title,currency,price,in_stock
iPhone 5c blue,USD,799,
iPad mini,USD,699,
"""
reader = smartcsv.reader(
StringIO(csv_data), columns=COLUMNS_WITH_VALUE_TRANSFORMATIONS)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': Decimal('799'),
'in_stock': ''
})
self.assertModelsEquals(ipad, {
'title': 'iPad mini',
'currency': 'USD',
'price': Decimal('699'),
'in_stock': ''
})
def test_required_value_is_missing_and_fail_fast(self):
"""Should fail fast and report the error"""
iphone_data = {'title': 'iPhone 5C'}
ipad_data = {'title': 'iPad mini', 'price': '699'}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row="{title},".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True
},
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('price' in e.errors)
else:
assert False, "Shouldn't reach this state"
def test_valid_data_and_skip_lines_with_header(self):
"""Should skip the first N lines and parse data ok with header"""
csv_data = """
Generated by Autobot 2000 - V0.1.2
----------
This next is intentionally left blank
-- Beginning of content
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA),
ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA),
)
reader = smartcsv.reader(
StringIO(csv_data), columns=COLUMNS_1, skip_lines=6)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_required_value_is_missing_and_dont_fail_fast(self):
"""Shouldn't fail fast, instead report errors on reader.errors"""
iphone_data = {'title': 'iPhone 5C'}
ipad_data = {'title': 'iPad mini', 'price': '699'}
iphone_row = "{title},".format(**iphone_data)
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row=iphone_row,
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True
},
],
fail_fast=False)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(ipad, dict))
self.assertModelsEquals(ipad, ipad_data)
self.assertTrue(reader.errors is not None)
self.assertTrue('rows' in reader.errors)
self.assertEqual(len(reader.errors['rows']), 1) # 1 row failing
self.assertRowError(reader.errors, iphone_row, 0, 'price')
def test_valid_csv_with_a_custom_dialect(self):
"""Should be valid if all data is passed"""
piped_template = VALID_TEMPLATE_STR.replace(',', '|')
csv_data = """
title|category|subcategory|currency|price|url|image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=piped_template.format(**IPHONE_DATA),
ipad_data=piped_template.format(**IPAD_DATA),
)
csv.register_dialect('pipes', delimiter='|')
reader = smartcsv.reader(
StringIO(csv_data), dialect='pipes', columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_not_required_value_is_provided(self):
"""Shouldn't fail and should have an the correct value"""
iphone_data = {'title': 'iPhone 5C', 'price': '799'}
ipad_data = {'title': 'iPad mini', 'price': '699'}
csv_data = """
title,price
{iphone_row}
{ipad_row}
""".format(iphone_row="{title},{price}".format(**iphone_data),
ipad_row="{title},{price}".format(**ipad_data))
reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'title',
'required': False
},
{
'name': 'price',
'required': False
},
])
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_valid_and_white_spaces_for_values_are_not_stripped(self):
"""Should strip white spaces out of values"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['category'] = " Phones "
ipad_data['price'] = " 599 "
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(
StringIO(csv_data), columns=COLUMNS_1, strip_white_spaces=False)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(
isinstance(iphone, dict) and isinstance(ipad, dict))
# First 4 spaces
iphone_data['title'] = " " + iphone_data['title']
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_valid_and_white_spaces_for_values_are_not_stripped(self):
"""Should strip white spaces out of values"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['category'] = " Phones "
ipad_data['price'] = " 599 "
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(StringIO(csv_data),
columns=COLUMNS_1,
strip_white_spaces=False)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
# First 4 spaces
iphone_data['title'] = " " + iphone_data['title']
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_column_required_and_choice_missing_with_fail_fast(self):
"""Should fail because the field is required"""
iphone_data = {
'title': 'iPhone 5C'
}
ipad_data = {
'title': 'iPad mini',
'currency': 'ARS'
}
csv_data = """
title,currency
{iphone_row}
{ipad_row}""".format(
iphone_row="{title},".format(**iphone_data),
ipad_row="{title},{currency}".format(**ipad_data)
)
reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'title', 'required': True},
{
'name': 'currency',
'required': True,
'choices': CURRENCY_CHOICES
}
])
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('currency' in e.errors)
else:
assert False, "Shouldn't reach this state"
def test_valid_csv_with_different_choices(self):
"""All choices for a field should be valid"""
iphone_data = IPHONE_DATA.copy()
ipad_data = IPAD_DATA.copy()
iphone_data['subcategory'] = ""
iphone_data['image_url'] = ""
ipad_data['subcategory'] = ""
ipad_data['image_url'] = ""
ipad_data['currency'] = "ARS"
csv_data = """
title,category,subcategory,currency,price,url,image_url
{iphone_data}
{ipad_data}
""".format(
iphone_data=VALID_TEMPLATE_STR.format(**iphone_data),
ipad_data=VALID_TEMPLATE_STR.format(**ipad_data),
)
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, iphone_data)
self.assertModelsEquals(ipad, ipad_data)
def test_invalid_header_fails_creating_the_reader(self):
"""Should fail to create the reader if the header is invalid"""
# Missing category header column
csv_data = """
title,subcategory,currency,price,url,image_url
iPhone 5c blue,Phones,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg
"""
self.assertRaises(InvalidCSVHeaderException, lambda: smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1))
def setUp(self):
csv_data = """
id,name
1,john
2,mary"""
self.reader = smartcsv.reader(StringIO(csv_data), columns=[
{'name': 'id', 'required': True},
{'name': 'name', 'required': True},
])
def test_default_value_doesnt_validate(self):
"""Should fail if the default value doesn't validate"""
columns = [
{"name": "title", "required": True},
{"name": "currency", "validator": is_number, "default": "INVALID"},
]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_default_value_not_in_choices(self):
"""Should fail if the default value not in choices"""
columns = [
{"name": "title", "required": True},
{"name": "currency", "choices": CURRENCY_CHOICES, "default": "INVALID"},
]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_invalid_header_fails_creating_the_reader(self):
"""Should fail to create the reader if the header is invalid"""
# Missing category header column
csv_data = """
title,subcategory,currency,price,url,image_url
iPhone 5c blue,Phones,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg
"""
self.assertRaises(
InvalidCSVHeaderException,
lambda: smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1))
def test_invalid_value_with_fail_fast_deactivated(self):
"""Shouldn't raise the exception raised by the transform function but save it in the errors attribute"""
invalid_row = "iPad mini,USD,INVALID,"
csv_data = """
title,currency,price,in_stock
iPhone 5c blue,USD,799,
{invalid_row}
Macbook Pro,USD,1399,yes
{invalid_row}
iPod shuffle,USD,199,
""".format(invalid_row=invalid_row)
reader = smartcsv.reader(StringIO(csv_data),
columns=self.columns,
fail_fast=False)
iphone = next(reader)
mac = next(reader)
ipod = next(reader)
self.assertModelsEquals(
iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': Decimal('799'),
'in_stock': ''
})
self.assertModelsEquals(
mac, {
'title': 'Macbook Pro',
'currency': 'USD',
'price': Decimal('1399'),
'in_stock': True
})
self.assertModelsEquals(
ipod, {
'title': 'iPod shuffle',
'currency': 'USD',
'price': Decimal('199'),
'in_stock': ''
})
self.assertTrue(reader.errors is not None)
self.assertTrue('rows' in reader.errors)
self.assertTrue(1 in reader.errors['rows'])
self.assertTrue(3 in reader.errors['rows'])
self.assertTrue('transform' in reader.errors['rows'][1]['errors'])
self.assertTrue('InvalidOperation' in reader.errors['rows'][1]
['errors']['transform'])
self.assertTrue('transform' in reader.errors['rows'][3]['errors'])
self.assertTrue('InvalidOperation' in reader.errors['rows'][3]
['errors']['transform'])
self.assertRowError(reader.errors, invalid_row, 1, 'transform')
def test_names_are_repeated(self):
"""Should fail a names are repeated"""
columns = [
{"name": "title", "required": True},
{"name": "title", "required": True},
{"name": "price", "required": True},
]
self.assertRaises(
InvalidCSVColumnDefinition,
lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False),
)
def test_fail_if_choices_doesnt_match(self):
"""Should fail if the provided value is not contained in the model choices"""
csv_data = """
title,category,subcategory,currency,price,url,image_url
iPhone 5c blue,Phones,Smartphones,INVALID,699,http://apple.com/iphone,http://apple.com/iphone.jpg
"""
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('currency' in e.errors)
def test_invalid_value_with_fail_fast_deactivated(self):
"""Shouldn't raise the exception raised by the transform function but save it in the errors attribute"""
invalid_row = "iPad mini,USD,INVALID,"
csv_data = """
title,currency,price,in_stock
iPhone 5c blue,USD,799,
{invalid_row}
Macbook Pro,USD,1399,yes
{invalid_row}
iPod shuffle,USD,199,
""".format(invalid_row=invalid_row)
reader = smartcsv.reader(
StringIO(csv_data), columns=self.columns, fail_fast=False)
iphone = next(reader)
mac = next(reader)
ipod = next(reader)
self.assertModelsEquals(iphone, {
'title': 'iPhone 5c blue',
'currency': 'USD',
'price': Decimal('799'),
'in_stock': ''
})
self.assertModelsEquals(mac, {
'title': 'Macbook Pro',
'currency': 'USD',
'price': Decimal('1399'),
'in_stock': True
})
self.assertModelsEquals(ipod, {
'title': 'iPod shuffle',
'currency': 'USD',
'price': Decimal('199'),
'in_stock': ''
})
self.assertTrue(reader.errors is not None)
self.assertTrue('rows' in reader.errors)
self.assertTrue(1 in reader.errors['rows'])
self.assertTrue(3 in reader.errors['rows'])
self.assertTrue('transform' in reader.errors['rows'][1]['errors'])
self.assertTrue(
'InvalidOperation' in
reader.errors['rows'][1]['errors']['transform'])
self.assertTrue('transform' in reader.errors['rows'][3]['errors'])
self.assertTrue(
'InvalidOperation' in
reader.errors['rows'][3]['errors']['transform'])
self.assertRowError(reader.errors, invalid_row, 1, 'transform')
def test_if_number_of_fields_is_invalid(self):
"""Should fail if the number of fields provided by the CSV is invalid"""
# Missing category field
csv_data = """
title,category,subcategory,currency,price,url,image_url
iPhone 5c blue,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg
"""
reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1)
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('row_length' in e.errors)
def test_csv_with_empty_rows_and_flag_enabled(self):
"""Should be valid and return the proper objects"""
reader = smartcsv.reader(StringIO(self.csv_data_with_empty_rows),
columns=COLUMNS_1, allow_empty_rows=True)
iphone = next(reader)
ipad = next(reader)
self.assertRaises(StopIteration, lambda: list(next(reader)))
self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict))
self.assertModelsEquals(iphone, IPHONE_DATA)
self.assertModelsEquals(ipad, IPAD_DATA)
def test_column_doesnt_have_a_name(self):
"""Should fail if a column doesn't have a name"""
columns = [
{
'required': True
},
{
'name': 'price',
'required': True
},
]
self.assertRaises(
InvalidCSVColumnDefinition, lambda: smartcsv.reader(
StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_doesnt_validate(self):
"""Should fail if the default value doesn't validate"""
columns = [
{
'name': 'title',
'required': True
},
{
'name': 'currency',
'validator': is_number,
'default': 'INVALID'
},
]
self.assertRaises(
InvalidCSVColumnDefinition, lambda: smartcsv.reader(
StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_is_empty(self):
"""Should fail a default value is empty"""
columns = [
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True,
'default': ''
},
]
self.assertRaises(
InvalidCSVColumnDefinition, lambda: smartcsv.reader(
StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_not_in_choices(self):
"""Should fail if the default value not in choices"""
columns = [
{
'name': 'title',
'required': True
},
{
'name': 'currency',
'choices': CURRENCY_CHOICES,
'default': 'INVALID'
},
]
self.assertRaises(
InvalidCSVColumnDefinition, lambda: smartcsv.reader(
StringIO(CSV_DATA), columns=columns, header_included=False))
def setUp(self):
csv_data = """
id,name
1,john
2,mary"""
self.reader = smartcsv.reader(StringIO(csv_data),
columns=[
{
'name': 'id',
'required': True
},
{
'name': 'name',
'required': True
},
])
def test_invalid_data_without_specifiying_a_header(self):
"""Should fail if the data is invalid and no header was provided."""
# Missing category field
csv_data = """
iPhone 5c blue,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg
"""
reader = smartcsv.reader(
StringIO(csv_data), columns=COLUMNS_1, header_included=False)
try:
next(reader)
except InvalidCSVException as e:
self.assertTrue(e.errors is not None)
self.assertTrue('row_length' in e.errors)
else:
assert False, "Shouldn't pass. Exception expected."
def test_column_validator_is_not_callable(self):
"""Should fail a validator is not callable"""
columns = [
{
'name': 'title',
'required': True
},
{
'name': 'price',
'required': True,
'validator': '8'
},
]
self.assertRaises(
InvalidCSVColumnDefinition, lambda: smartcsv.reader(
StringIO(CSV_DATA), columns=columns, header_included=False))
