def test_required_column_empty_fails_even_with_default(self): """Should fail if the column is required and the value is empty.""" iphone_data = {'title': 'iPhone 5C', 'price': ''} ipad_data = {'title': 'iPad mini', 'price': '799'} csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True, 'default': 999 }, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors)
def test_required_column_with_data_is_transformed(self): """Should apply the transformation to the value""" iphone_data = {"title": "iPhone 5C", "price": "799"} ipad_data = {"title": "iPad mini", "price": "699"} csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader( StringIO(csv_data), columns=[ {"name": "title", "required": True}, {"name": "price", "required": True, "transform": lambda x: Decimal(x)}, ], ) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, {"title": "iPhone 5C", "price": Decimal("799")}) self.assertModelsEquals(ipad, {"title": "iPad mini", "price": Decimal("699")})
def test_default_value_is_not_transformed(self): """Shouldn't apply no transformation if the value is missing and the default value is being used""" iphone_data = {"title": "iPhone 5C", "price": ""} ipad_data = {"title": "iPad mini", "price": ""} csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) mocked_validator = mock.MagicMock(return_value=True) reader = smartcsv.reader( StringIO(csv_data), columns=[ {"name": "title", "required": True}, {"name": "price", "required": False, "transform": mocked_validator, "default": 899}, ], ) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, {"title": "iPhone 5C", "price": 899}) self.assertModelsEquals(ipad, {"title": "iPad mini", "price": 899}) self.assertEqual(mocked_validator.call_count, 0)
def test_error_exception_is_reported_without_fail_fast(self): """reader.errors should contain the exception that happenend with the value transformation""" iphone_data = {"title": "iPhone 5C", "price": "INVALID"} ipad_data = {"title": "iPad mini", "price": "699"} iphone_row = "{title},{price}".format(**iphone_data) csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row=iphone_row, ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader( StringIO(csv_data), columns=[ {"name": "title", "required": True}, {"name": "price", "required": True, "transform": lambda x: Decimal(x)}, ], fail_fast=False, ) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(ipad, dict)) self.assertModelsEquals(ipad, {"title": "iPad mini", "price": Decimal("699")}) self.assertTrue(reader.errors is not None) self.assertTrue("rows" in reader.errors) self.assertEqual(len(reader.errors["rows"]), 1) # 1 row failing self.assertRowError(reader.errors, iphone_row, 0, "transform")
def test_not_required_value_is_provided(self): """Shouldn't fail and should have an the correct value""" iphone_data = { 'title': 'iPhone 5C', 'price': '799' } ipad_data = { 'title': 'iPad mini', 'price': '699' } csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': False}, {'name': 'price', 'required': False}, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_required_value_is_missing_and_dont_fail_fast(self): """Shouldn't fail fast, instead report errors on reader.errors""" iphone_data = { 'title': 'iPhone 5C' } ipad_data = { 'title': 'iPad mini', 'price': '699' } iphone_row = "{title},".format(**iphone_data) csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row=iphone_row, ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, {'name': 'price', 'required': True}, ], fail_fast=False) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(ipad, dict)) self.assertModelsEquals(ipad, ipad_data) self.assertTrue(reader.errors is not None) self.assertTrue('rows' in reader.errors) self.assertEqual(len(reader.errors['rows']), 1) # 1 row failing self.assertRowError( reader.errors, iphone_row, 0, 'price')
def test_not_required_column_empty_returns_default_value(self): """Should return the default value if no other is provided""" iphone_data = {'title': 'iPhone 5C', 'price': ''} ipad_data = {'title': 'iPad mini', 'price': '799'} csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': False, 'default': 999 }, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, {'title': 'iPhone 5C', 'price': 999}) self.assertModelsEquals(ipad, ipad_data)
def test_valid_csv_with_blank_lines(self): """Should be valid even if there are blank lines""" csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA), ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA), ) reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def test_invalid_value_is_passed_and_exception_is_raised(self): """Should not validate and raise a exception (fail_fast=True)""" iphone_data = { 'title': 'iPhone 5C', 'price': 'INVALID' } ipad_data = { 'title': 'iPad mini', 'price': '699' } csv_data = """ title,price {iphone_row} {ipad_row}""".format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, { 'name': 'price', 'required': True, 'validator': is_number }, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors)
def test_valid_csv_with_blank_lines(self): """Should be valid even if there are blank lines""" csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA), ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA), ) reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue( isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def handle(self, *args, **options): if len(args) != 1: raise CommandError('Usage: importproducts ' + self.args) filepath = args[0] print 'Importing persons from {0}...'.format(filepath) stats = {'imported': 0, 'skipped': 0} with open(filepath) as f: reader = smartcsv.reader(f, columns=COLUMNS, fail_fast=False) for person in reader: try: Profile.objects.create( email=person['email'], lg_full_name=person['name'], lg_timezone=person['timezone'] ) except IntegrityError: stats['skipped'] += 1 else: stats['imported'] += 1 print ('Finished with stats: ' 'imported {imported}, skipped {skipped}').format(**stats)
def test_choice_field_is_required_and_missing_value_is_invalid(self): """Should fail to validate if a required choice field is missing""" csv_data = """ title,currency,price iPhone 5c blue,USD,799 iPad mini,,699 """ columns = [{ 'name': 'title', 'required': True }, { 'name': 'currency', 'required': True, 'choices': CURRENCY_CHOICES }, { 'name': 'price', 'required': True, 'validator': is_number }] reader = smartcsv.reader(StringIO(csv_data), columns=columns) iphone = next(reader) self.assertRaises(InvalidCSVException, lambda: next(reader)) self.assertModelsEquals(iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': '799' })
def test_valid_csv_with_some_fields_not_required_empty(self): """Should be valid if some not required fields are filled and others don't""" iphone_data = IPHONE_DATA.copy() ipad_data = IPAD_DATA.copy() iphone_data['subcategory'] = "" iphone_data['image_url'] = "" ipad_data['image_url'] = "" csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**iphone_data), ipad_data=VALID_TEMPLATE_STR.format(**ipad_data), ) reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertEqual(iphone['title'], 'iPhone 5c blue') self.assertEqual(ipad['title'], 'iPad mini') self.assertEqual(iphone['subcategory'], '') self.assertEqual(ipad['subcategory'], 'Apple')
def test_valid_csv_with_columns_names_with_whitespaces(self): """Should be valid if all data is passed""" csv_data = """ title,category,subcategory,currency,price,url,Image URL {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA), ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA), ) columns = COLUMNS_1[:] image_column = columns[-1].copy() image_column['name'] = "Image URL" columns = columns[:-1] + [image_column] reader = smartcsv.reader(StringIO(csv_data), columns=columns) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertEqual(iphone['Image URL'], 'http://apple.com/iphone.jpg') self.assertEqual(ipad['Image URL'], 'http://apple.com/ipad.jpg')
def test_choice_field_is_required_and_missing_value_is_invalid(self): """Should fail to validate if a required choice field is missing""" csv_data = """ title,currency,price iPhone 5c blue,USD,799 iPad mini,,699 """ columns = [ {'name': 'title', 'required': True}, { 'name': 'currency', 'required': True, 'choices': CURRENCY_CHOICES }, { 'name': 'price', 'required': True, 'validator': is_number } ] reader = smartcsv.reader(StringIO(csv_data), columns=columns) iphone = next(reader) self.assertRaises(InvalidCSVException, lambda: next(reader)) self.assertModelsEquals(iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': '799' })
def test_skip_column_doesnt_invoke_validator(self): """Should return the object without the skipped column""" iphone_data = {'title': 'iPhone 5C', 'price': '799'} ipad_data = {'title': 'iPad mini', 'price': '699'} csv_data = """ title,price {iphone_row} {ipad_row}""".format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) mocked_validator = mock.MagicMock(return_value=True) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'validator': mocked_validator, 'skip': True }, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, {'title': 'iPhone 5C'}) self.assertModelsEquals(ipad, {'title': 'iPad mini'}) self.assertEqual(mocked_validator.call_count, 0)
def test_column_validator_is_not_callable(self): """Should fail a validator is not callable""" columns = [{"name": "title", "required": True}, {"name": "price", "required": True, "validator": "8"}] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_column_doesnt_have_a_name(self): """Should fail if a column doesn't have a name""" columns = [{"required": True}, {"name": "price", "required": True}] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_default_value_is_empty(self): """Should fail a default value is empty""" columns = [{"name": "title", "required": True}, {"name": "price", "required": True, "default": ""}] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_valid_and_value_transformed_with_only_required_data(self): """Should transform all values with only required data present""" csv_data = """ title,currency,price,in_stock iPhone 5c blue,USD,799, iPad mini,USD,699, """ reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_WITH_VALUE_TRANSFORMATIONS) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals( iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': Decimal('799'), 'in_stock': '' }) self.assertModelsEquals( ipad, { 'title': 'iPad mini', 'currency': 'USD', 'price': Decimal('699'), 'in_stock': '' })
def test_column_with_missing_data_is_not_transformed(self): """Shouldn't invoke the transform function if no value is passed""" iphone_data = {'title': 'iPhone 5C', 'price': ''} ipad_data = {'title': 'iPad mini', 'price': ''} csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) mocked_validator = mock.MagicMock(return_value=True) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': False, 'transform': mocked_validator }, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data) self.assertEqual(mocked_validator.call_count, 0)
def test_valid_choice_is_provided(self): """Should not fail and have the value of the selected choice""" iphone_data = { 'title': 'iPhone 5C', 'currency': 'USD' } ipad_data = { 'title': 'iPad mini', 'currency': 'ARS' } csv_data = """ title,currency {iphone_row} {ipad_row}""".format( iphone_row="{title},{currency}".format(**iphone_data), ipad_row="{title},{currency}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, { 'name': 'currency', 'required': True, 'choices': CURRENCY_CHOICES } ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_not_required_column_empty_returns_default_value(self): """Should return the default value if no other is provided""" iphone_data = { 'title': 'iPhone 5C', 'price': '' } ipad_data = { 'title': 'iPad mini', 'price': '799' } csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, {'name': 'price', 'required': False, 'default': 999}, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, { 'title': 'iPhone 5C', 'price': 999 }) self.assertModelsEquals(ipad, ipad_data)
def test_valid_csv_with_different_choices(self): """All choices for a field should be valid""" iphone_data = IPHONE_DATA.copy() ipad_data = IPAD_DATA.copy() iphone_data['subcategory'] = "" iphone_data['image_url'] = "" ipad_data['subcategory'] = "" ipad_data['image_url'] = "" ipad_data['currency'] = "ARS" csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**iphone_data), ipad_data=VALID_TEMPLATE_STR.format(**ipad_data), ) reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_required_column_empty_fails_even_with_default(self): """Should fail if the column is required and the value is empty.""" iphone_data = { 'title': 'iPhone 5C', 'price': '' } ipad_data = { 'title': 'iPad mini', 'price': '799' } csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, {'name': 'price', 'required': True, 'default': 999}, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors)
def test_valid_csv_with_a_custom_dialect(self): """Should be valid if all data is passed""" piped_template = VALID_TEMPLATE_STR.replace(',', '|') csv_data = """ title|category|subcategory|currency|price|url|image_url {iphone_data} {ipad_data} """.format( iphone_data=piped_template.format(**IPHONE_DATA), ipad_data=piped_template.format(**IPAD_DATA), ) csv.register_dialect('pipes', delimiter='|') reader = smartcsv.reader(StringIO(csv_data), dialect='pipes', columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def test_valid_data_and_skip_lines_with_header(self): """Should skip the first N lines and parse data ok with header""" csv_data = """ Generated by Autobot 2000 - V0.1.2 ---------- This next is intentionally left blank -- Beginning of content title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**IPHONE_DATA), ipad_data=VALID_TEMPLATE_STR.format(**IPAD_DATA), ) reader = smartcsv.reader( StringIO(csv_data), columns=COLUMNS_1, skip_lines=6) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue( isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def test_required_value_is_missing_and_fail_fast(self): """Should fail fast and report the error""" iphone_data = { 'title': 'iPhone 5C' } ipad_data = { 'title': 'iPad mini', 'price': '699' } csv_data = """ title,price {iphone_row} {ipad_row} """.format( iphone_row="{title},".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, {'name': 'price', 'required': True}, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors) else: assert False, "Shouldn't reach this state"
def test_invalid_value_is_passed_and_exception_is_raised(self): """Should not validate and raise a exception (fail_fast=True)""" iphone_data = {'title': 'iPhone 5C', 'price': 'INVALID'} ipad_data = {'title': 'iPad mini', 'price': '699'} csv_data = """ title,price {iphone_row} {ipad_row}""".format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True, 'validator': is_number }, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors)
def test_valid_and_value_transformed_with_only_required_data(self): """Should transform all values with only required data present""" csv_data = """ title,currency,price,in_stock iPhone 5c blue,USD,799, iPad mini,USD,699, """ reader = smartcsv.reader( StringIO(csv_data), columns=COLUMNS_WITH_VALUE_TRANSFORMATIONS) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue( isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': Decimal('799'), 'in_stock': '' }) self.assertModelsEquals(ipad, { 'title': 'iPad mini', 'currency': 'USD', 'price': Decimal('699'), 'in_stock': '' })
def test_required_value_is_missing_and_fail_fast(self): """Should fail fast and report the error""" iphone_data = {'title': 'iPhone 5C'} ipad_data = {'title': 'iPad mini', 'price': '699'} csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row="{title},".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True }, ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('price' in e.errors) else: assert False, "Shouldn't reach this state"
def test_required_value_is_missing_and_dont_fail_fast(self): """Shouldn't fail fast, instead report errors on reader.errors""" iphone_data = {'title': 'iPhone 5C'} ipad_data = {'title': 'iPad mini', 'price': '699'} iphone_row = "{title},".format(**iphone_data) csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row=iphone_row, ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True }, ], fail_fast=False) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(ipad, dict)) self.assertModelsEquals(ipad, ipad_data) self.assertTrue(reader.errors is not None) self.assertTrue('rows' in reader.errors) self.assertEqual(len(reader.errors['rows']), 1) # 1 row failing self.assertRowError(reader.errors, iphone_row, 0, 'price')
def test_valid_csv_with_a_custom_dialect(self): """Should be valid if all data is passed""" piped_template = VALID_TEMPLATE_STR.replace(',', '|') csv_data = """ title|category|subcategory|currency|price|url|image_url {iphone_data} {ipad_data} """.format( iphone_data=piped_template.format(**IPHONE_DATA), ipad_data=piped_template.format(**IPAD_DATA), ) csv.register_dialect('pipes', delimiter='|') reader = smartcsv.reader( StringIO(csv_data), dialect='pipes', columns=COLUMNS_1) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue( isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def test_not_required_value_is_provided(self): """Shouldn't fail and should have an the correct value""" iphone_data = {'title': 'iPhone 5C', 'price': '799'} ipad_data = {'title': 'iPad mini', 'price': '699'} csv_data = """ title,price {iphone_row} {ipad_row} """.format(iphone_row="{title},{price}".format(**iphone_data), ipad_row="{title},{price}".format(**ipad_data)) reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'title', 'required': False }, { 'name': 'price', 'required': False }, ]) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_valid_and_white_spaces_for_values_are_not_stripped(self): """Should strip white spaces out of values""" iphone_data = IPHONE_DATA.copy() ipad_data = IPAD_DATA.copy() iphone_data['category'] = " Phones " ipad_data['price'] = " 599 " csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**iphone_data), ipad_data=VALID_TEMPLATE_STR.format(**ipad_data), ) reader = smartcsv.reader( StringIO(csv_data), columns=COLUMNS_1, strip_white_spaces=False) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue( isinstance(iphone, dict) and isinstance(ipad, dict)) # First 4 spaces iphone_data['title'] = " " + iphone_data['title'] self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_valid_and_white_spaces_for_values_are_not_stripped(self): """Should strip white spaces out of values""" iphone_data = IPHONE_DATA.copy() ipad_data = IPAD_DATA.copy() iphone_data['category'] = " Phones " ipad_data['price'] = " 599 " csv_data = """ title,category,subcategory,currency,price,url,image_url {iphone_data} {ipad_data} """.format( iphone_data=VALID_TEMPLATE_STR.format(**iphone_data), ipad_data=VALID_TEMPLATE_STR.format(**ipad_data), ) reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1, strip_white_spaces=False) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) # First 4 spaces iphone_data['title'] = " " + iphone_data['title'] self.assertModelsEquals(iphone, iphone_data) self.assertModelsEquals(ipad, ipad_data)
def test_column_required_and_choice_missing_with_fail_fast(self): """Should fail because the field is required""" iphone_data = { 'title': 'iPhone 5C' } ipad_data = { 'title': 'iPad mini', 'currency': 'ARS' } csv_data = """ title,currency {iphone_row} {ipad_row}""".format( iphone_row="{title},".format(**iphone_data), ipad_row="{title},{currency}".format(**ipad_data) ) reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'title', 'required': True}, { 'name': 'currency', 'required': True, 'choices': CURRENCY_CHOICES } ]) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('currency' in e.errors) else: assert False, "Shouldn't reach this state"
def test_invalid_header_fails_creating_the_reader(self): """Should fail to create the reader if the header is invalid""" # Missing category header column csv_data = """ title,subcategory,currency,price,url,image_url iPhone 5c blue,Phones,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg """ self.assertRaises(InvalidCSVHeaderException, lambda: smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1))
def setUp(self): csv_data = """ id,name 1,john 2,mary""" self.reader = smartcsv.reader(StringIO(csv_data), columns=[ {'name': 'id', 'required': True}, {'name': 'name', 'required': True}, ])
def test_default_value_doesnt_validate(self): """Should fail if the default value doesn't validate""" columns = [ {"name": "title", "required": True}, {"name": "currency", "validator": is_number, "default": "INVALID"}, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_default_value_not_in_choices(self): """Should fail if the default value not in choices""" columns = [ {"name": "title", "required": True}, {"name": "currency", "choices": CURRENCY_CHOICES, "default": "INVALID"}, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_invalid_header_fails_creating_the_reader(self): """Should fail to create the reader if the header is invalid""" # Missing category header column csv_data = """ title,subcategory,currency,price,url,image_url iPhone 5c blue,Phones,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg """ self.assertRaises( InvalidCSVHeaderException, lambda: smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1))
def test_invalid_value_with_fail_fast_deactivated(self): """Shouldn't raise the exception raised by the transform function but save it in the errors attribute""" invalid_row = "iPad mini,USD,INVALID," csv_data = """ title,currency,price,in_stock iPhone 5c blue,USD,799, {invalid_row} Macbook Pro,USD,1399,yes {invalid_row} iPod shuffle,USD,199, """.format(invalid_row=invalid_row) reader = smartcsv.reader(StringIO(csv_data), columns=self.columns, fail_fast=False) iphone = next(reader) mac = next(reader) ipod = next(reader) self.assertModelsEquals( iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': Decimal('799'), 'in_stock': '' }) self.assertModelsEquals( mac, { 'title': 'Macbook Pro', 'currency': 'USD', 'price': Decimal('1399'), 'in_stock': True }) self.assertModelsEquals( ipod, { 'title': 'iPod shuffle', 'currency': 'USD', 'price': Decimal('199'), 'in_stock': '' }) self.assertTrue(reader.errors is not None) self.assertTrue('rows' in reader.errors) self.assertTrue(1 in reader.errors['rows']) self.assertTrue(3 in reader.errors['rows']) self.assertTrue('transform' in reader.errors['rows'][1]['errors']) self.assertTrue('InvalidOperation' in reader.errors['rows'][1] ['errors']['transform']) self.assertTrue('transform' in reader.errors['rows'][3]['errors']) self.assertTrue('InvalidOperation' in reader.errors['rows'][3] ['errors']['transform']) self.assertRowError(reader.errors, invalid_row, 1, 'transform')
def test_names_are_repeated(self): """Should fail a names are repeated""" columns = [ {"name": "title", "required": True}, {"name": "title", "required": True}, {"name": "price", "required": True}, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader(StringIO(CSV_DATA), columns=columns, header_included=False), )
def test_fail_if_choices_doesnt_match(self): """Should fail if the provided value is not contained in the model choices""" csv_data = """ title,category,subcategory,currency,price,url,image_url iPhone 5c blue,Phones,Smartphones,INVALID,699,http://apple.com/iphone,http://apple.com/iphone.jpg """ reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('currency' in e.errors)
def test_invalid_value_with_fail_fast_deactivated(self): """Shouldn't raise the exception raised by the transform function but save it in the errors attribute""" invalid_row = "iPad mini,USD,INVALID," csv_data = """ title,currency,price,in_stock iPhone 5c blue,USD,799, {invalid_row} Macbook Pro,USD,1399,yes {invalid_row} iPod shuffle,USD,199, """.format(invalid_row=invalid_row) reader = smartcsv.reader( StringIO(csv_data), columns=self.columns, fail_fast=False) iphone = next(reader) mac = next(reader) ipod = next(reader) self.assertModelsEquals(iphone, { 'title': 'iPhone 5c blue', 'currency': 'USD', 'price': Decimal('799'), 'in_stock': '' }) self.assertModelsEquals(mac, { 'title': 'Macbook Pro', 'currency': 'USD', 'price': Decimal('1399'), 'in_stock': True }) self.assertModelsEquals(ipod, { 'title': 'iPod shuffle', 'currency': 'USD', 'price': Decimal('199'), 'in_stock': '' }) self.assertTrue(reader.errors is not None) self.assertTrue('rows' in reader.errors) self.assertTrue(1 in reader.errors['rows']) self.assertTrue(3 in reader.errors['rows']) self.assertTrue('transform' in reader.errors['rows'][1]['errors']) self.assertTrue( 'InvalidOperation' in reader.errors['rows'][1]['errors']['transform']) self.assertTrue('transform' in reader.errors['rows'][3]['errors']) self.assertTrue( 'InvalidOperation' in reader.errors['rows'][3]['errors']['transform']) self.assertRowError(reader.errors, invalid_row, 1, 'transform')
def test_if_number_of_fields_is_invalid(self): """Should fail if the number of fields provided by the CSV is invalid""" # Missing category field csv_data = """ title,category,subcategory,currency,price,url,image_url iPhone 5c blue,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg """ reader = smartcsv.reader(StringIO(csv_data), columns=COLUMNS_1) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('row_length' in e.errors)
def test_csv_with_empty_rows_and_flag_enabled(self): """Should be valid and return the proper objects""" reader = smartcsv.reader(StringIO(self.csv_data_with_empty_rows), columns=COLUMNS_1, allow_empty_rows=True) iphone = next(reader) ipad = next(reader) self.assertRaises(StopIteration, lambda: list(next(reader))) self.assertTrue(isinstance(iphone, dict) and isinstance(ipad, dict)) self.assertModelsEquals(iphone, IPHONE_DATA) self.assertModelsEquals(ipad, IPAD_DATA)
def test_column_doesnt_have_a_name(self): """Should fail if a column doesn't have a name""" columns = [ { 'required': True }, { 'name': 'price', 'required': True }, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader( StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_doesnt_validate(self): """Should fail if the default value doesn't validate""" columns = [ { 'name': 'title', 'required': True }, { 'name': 'currency', 'validator': is_number, 'default': 'INVALID' }, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader( StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_is_empty(self): """Should fail a default value is empty""" columns = [ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True, 'default': '' }, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader( StringIO(CSV_DATA), columns=columns, header_included=False))
def test_default_value_not_in_choices(self): """Should fail if the default value not in choices""" columns = [ { 'name': 'title', 'required': True }, { 'name': 'currency', 'choices': CURRENCY_CHOICES, 'default': 'INVALID' }, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader( StringIO(CSV_DATA), columns=columns, header_included=False))
def setUp(self): csv_data = """ id,name 1,john 2,mary""" self.reader = smartcsv.reader(StringIO(csv_data), columns=[ { 'name': 'id', 'required': True }, { 'name': 'name', 'required': True }, ])
def test_invalid_data_without_specifiying_a_header(self): """Should fail if the data is invalid and no header was provided.""" # Missing category field csv_data = """ iPhone 5c blue,Smartphones,USD,699,http://apple.com/iphone,http://apple.com/iphone.jpg """ reader = smartcsv.reader( StringIO(csv_data), columns=COLUMNS_1, header_included=False) try: next(reader) except InvalidCSVException as e: self.assertTrue(e.errors is not None) self.assertTrue('row_length' in e.errors) else: assert False, "Shouldn't pass. Exception expected."
def test_column_validator_is_not_callable(self): """Should fail a validator is not callable""" columns = [ { 'name': 'title', 'required': True }, { 'name': 'price', 'required': True, 'validator': '8' }, ] self.assertRaises( InvalidCSVColumnDefinition, lambda: smartcsv.reader( StringIO(CSV_DATA), columns=columns, header_included=False))