def test_write_problems():
"""Test writing problems as restructured text."""
class MockFile(object):
def __init__(self):
self.content = ''
def write(self, s):
self.content += s
file = MockFile()
problems = [{
'code': 'X1',
'message': 'invalid foo',
'row': 2,
'field': 'foo',
'context': {
'info': 'interesting'
}
}, {
'code': 'X2',
'message': 'invalid bar',
'row': 3,
'field': 'bar',
'context': {
'info': 'very interesting'
}
}]
expectation = """
=================
Validation Report
=================
Problems
========
X1 - invalid foo
----------------
:field: foo
:row: 2
:info: interesting
X2 - invalid bar
----------------
:field: bar
:row: 3
:info: very interesting
Summary
=======
Found 2 problems in total.
:X1: 1
:X2: 1
"""
write_problems(problems, file)
assert file.content == expectation, file.content
def _validate_csv(csv_file, output_file=None):
"""
Validates a CSV file.
:param csv_file: The CSV file to validate
:param output_file: The optional output file to which problems should
be written
:returns: True if the CSV file is valid, false otherwise
"""
field_names = _get_header(csv_file)
validator = CSVValidator(field_names)
# basic header and record length checks
validator.add_header_check('EX1', 'bad header')
validator.add_record_length_check('EX2', 'unexpected record length')
with open(csv_file) as fp:
data = csv.reader(fp)
problems = validator.validate(data)
if problems:
write_problems(problems, output_file or sys.stdout)
return False
else:
return True
def test_write_problems_summarize():
"""Test writing a problem summary as restructured text."""
class MockFile(object):
def __init__(self):
self.content = ''
def write(self, s):
self.content += s
file = MockFile()
problems = [
{
'code': 'X1',
'message': 'invalid foo',
'row': 2,
'field': 'foo',
'context': {
'info': 'interesting'
}
},
{
'code': 'X2',
'message': 'invalid bar',
'row': 3,
'field': 'bar',
'context': {
'info': 'very interesting'
}
},
{
'code': 'X2',
'message': 'invalid bar',
'row': 4,
'field': 'bar',
'context': {
'info': 'very very interesting'
}
}
]
expectation = """
=================
Validation Report
=================
Summary
=======
Found 3 problems in total.
:X1: 1
:X2: 2
"""
write_problems(problems, file, summarize=True)
assert file.content == expectation, file.content
def test_write_problems_summarize():
"""Test writing a problem summary as restructured text."""
class MockFile(object):
def __init__(self):
self.content = ''
def write(self, s):
self.content += s
file = MockFile()
problems = [
{
'code': 'X1',
'message': 'invalid foo',
'row': 2,
'field': 'foo',
'context': {
'info': 'interesting'
}
},
{
'code': 'X2',
'message': 'invalid bar',
'row': 3,
'field': 'bar',
'context': {
'info': 'very interesting'
}
},
{
'code': 'X2',
'message': 'invalid bar',
'row': 4,
'field': 'bar',
'context': {
'info': 'very very interesting'
}
}
]
expectation = """
=================
Validation Report
=================
Summary
=======
Found 3 problems in total.
:X1: 1
:X2: 2
"""
write_problems(problems, file, summarize=True)
assert file.content == expectation, file.content
def test_write_problems_with_limit():
"""Test writing problems with a limit as restructured text."""
class MockFile(object):
def __init__(self):
self.content = ''
def write(self, s):
self.content += s
file = MockFile()
problems = [
{
'code': 'X1',
'message': 'invalid foo',
'row': 2,
'field': 'foo',
'context': {
'info': 'interesting'
}
},
{
'code': 'X2',
'message': 'invalid bar',
'row': 3,
'field': 'bar',
'context': {
'info': 'very interesting'
}
}
]
expectation = """
=================
Validation Report
=================
Problems
========
X1 - invalid foo
----------------
:field: foo
:row: 2
:info: interesting
Summary
=======
Found at least 1 problem in total.
:X1: 1
"""
write_problems(problems, file, limit=1)
assert file.content == expectation, file.content
def main():
"""Main function."""
# define a command-line argument parser
description = 'Validate a CSV data file.'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('file',
metavar='FILE',
help='a file to be validated')
parser.add_argument('-l', '--limit',
dest='limit',
type=int,
action='store',
default=0,
help='limit the number of problems reported'
)
parser.add_argument('-s', '--summarize',
dest='summarize',
action='store_true',
default=False,
help='output only a summary of the different types of problem found'
)
parser.add_argument('-e', '--report-unexpected-exceptions',
dest='report_unexpected_exceptions',
action='store_true',
default=False,
help='report any unexpected exceptions as problems'
)
# parse arguments
args = parser.parse_args()
# sanity check arguments
if not os.path.isfile(args.file):
print('%s is not a file' % args.file)
sys.exit(1)
with open(args.file, 'r') as f:
# set up a csv reader for the data
data = csv.reader(f, delimiter='\t')
# create a validator
validator = create_validator()
# validate the data from the csv reader
# N.B., validate() returns a list of problems;
# if you expect a large number of problems, use ivalidate() instead
# of validate(), but bear in mind that ivalidate() returns an iterator
# so there is no len()
problems = validator.validate(data,
summarize=args.summarize,
report_unexpected_exceptions=args.report_unexpected_exceptions,
context={'file': args.file})
# write problems to stdout as restructured text
write_problems(problems, sys.stdout,
summarize=args.summarize,
limit=args.limit)
# decide how to exit
if problems: # will not work with ivalidate() because it returns an iterator
sys.exit(1)
else:
sys.exit(0)
def main():
"""Main function."""
# define a command-line argument parser
description = 'Validate a CSV data file.'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('file',
metavar='FILE',
help='a file to be validated')
parser.add_argument('-l', '--limit',
dest='limit',
type=int,
action='store',
default=0,
help='limit the number of problems reported'
)
parser.add_argument('-s', '--summarize',
dest='summarize',
action='store_true',
default=False,
help='output only a summary of the different types of problem found'
)
parser.add_argument('-e', '--report-unexpected-exceptions',
dest='report_unexpected_exceptions',
action='store_true',
default=False,
help='report any unexpected exceptions as problems'
)
# parse arguments
args = parser.parse_args()
# sanity check arguments
if not os.path.isfile(args.file):
print '%s is not a file' % args.file
sys.exit(1)
with open(args.file, 'r') as f:
# set up a csv reader for the data
data = csv.reader(f, delimiter='\t')
# create a validator
validator = create_validator()
# validate the data from the csv reader
# N.B., validate() returns a list of problems;
# if you expect a large number of problems, use ivalidate() instead
# of validate(), but bear in mind that ivalidate() returns an iterator
# so there is no len()
problems = validator.validate(data,
summarize=args.summarize,
report_unexpected_exceptions=args.report_unexpected_exceptions,
context={'file': args.file})
# write problems to stdout as restructured text
write_problems(problems, sys.stdout,
summarize=args.summarize,
limit=args.limit)
# decide how to exit
if problems: # will not work with ivalidate() because it returns an iterator
sys.exit(1)
else:
sys.exit(0)
