def test_write_problems(): """Test writing problems as restructured text.""" class MockFile(object): def __init__(self): self.content = '' def write(self, s): self.content += s file = MockFile() problems = [{ 'code': 'X1', 'message': 'invalid foo', 'row': 2, 'field': 'foo', 'context': { 'info': 'interesting' } }, { 'code': 'X2', 'message': 'invalid bar', 'row': 3, 'field': 'bar', 'context': { 'info': 'very interesting' } }] expectation = """ ================= Validation Report ================= Problems ======== X1 - invalid foo ---------------- :field: foo :row: 2 :info: interesting X2 - invalid bar ---------------- :field: bar :row: 3 :info: very interesting Summary ======= Found 2 problems in total. :X1: 1 :X2: 1 """ write_problems(problems, file) assert file.content == expectation, file.content
def _validate_csv(csv_file, output_file=None): """ Validates a CSV file. :param csv_file: The CSV file to validate :param output_file: The optional output file to which problems should be written :returns: True if the CSV file is valid, false otherwise """ field_names = _get_header(csv_file) validator = CSVValidator(field_names) # basic header and record length checks validator.add_header_check('EX1', 'bad header') validator.add_record_length_check('EX2', 'unexpected record length') with open(csv_file) as fp: data = csv.reader(fp) problems = validator.validate(data) if problems: write_problems(problems, output_file or sys.stdout) return False else: return True
def test_write_problems_summarize(): """Test writing a problem summary as restructured text.""" class MockFile(object): def __init__(self): self.content = '' def write(self, s): self.content += s file = MockFile() problems = [ { 'code': 'X1', 'message': 'invalid foo', 'row': 2, 'field': 'foo', 'context': { 'info': 'interesting' } }, { 'code': 'X2', 'message': 'invalid bar', 'row': 3, 'field': 'bar', 'context': { 'info': 'very interesting' } }, { 'code': 'X2', 'message': 'invalid bar', 'row': 4, 'field': 'bar', 'context': { 'info': 'very very interesting' } } ] expectation = """ ================= Validation Report ================= Summary ======= Found 3 problems in total. :X1: 1 :X2: 2 """ write_problems(problems, file, summarize=True) assert file.content == expectation, file.content
def test_write_problems_with_limit(): """Test writing problems with a limit as restructured text.""" class MockFile(object): def __init__(self): self.content = '' def write(self, s): self.content += s file = MockFile() problems = [ { 'code': 'X1', 'message': 'invalid foo', 'row': 2, 'field': 'foo', 'context': { 'info': 'interesting' } }, { 'code': 'X2', 'message': 'invalid bar', 'row': 3, 'field': 'bar', 'context': { 'info': 'very interesting' } } ] expectation = """ ================= Validation Report ================= Problems ======== X1 - invalid foo ---------------- :field: foo :row: 2 :info: interesting Summary ======= Found at least 1 problem in total. :X1: 1 """ write_problems(problems, file, limit=1) assert file.content == expectation, file.content
def main(): """Main function.""" # define a command-line argument parser description = 'Validate a CSV data file.' parser = argparse.ArgumentParser(description=description) parser.add_argument('file', metavar='FILE', help='a file to be validated') parser.add_argument('-l', '--limit', dest='limit', type=int, action='store', default=0, help='limit the number of problems reported' ) parser.add_argument('-s', '--summarize', dest='summarize', action='store_true', default=False, help='output only a summary of the different types of problem found' ) parser.add_argument('-e', '--report-unexpected-exceptions', dest='report_unexpected_exceptions', action='store_true', default=False, help='report any unexpected exceptions as problems' ) # parse arguments args = parser.parse_args() # sanity check arguments if not os.path.isfile(args.file): print('%s is not a file' % args.file) sys.exit(1) with open(args.file, 'r') as f: # set up a csv reader for the data data = csv.reader(f, delimiter='\t') # create a validator validator = create_validator() # validate the data from the csv reader # N.B., validate() returns a list of problems; # if you expect a large number of problems, use ivalidate() instead # of validate(), but bear in mind that ivalidate() returns an iterator # so there is no len() problems = validator.validate(data, summarize=args.summarize, report_unexpected_exceptions=args.report_unexpected_exceptions, context={'file': args.file}) # write problems to stdout as restructured text write_problems(problems, sys.stdout, summarize=args.summarize, limit=args.limit) # decide how to exit if problems: # will not work with ivalidate() because it returns an iterator sys.exit(1) else: sys.exit(0)
def main(): """Main function.""" # define a command-line argument parser description = 'Validate a CSV data file.' parser = argparse.ArgumentParser(description=description) parser.add_argument('file', metavar='FILE', help='a file to be validated') parser.add_argument('-l', '--limit', dest='limit', type=int, action='store', default=0, help='limit the number of problems reported' ) parser.add_argument('-s', '--summarize', dest='summarize', action='store_true', default=False, help='output only a summary of the different types of problem found' ) parser.add_argument('-e', '--report-unexpected-exceptions', dest='report_unexpected_exceptions', action='store_true', default=False, help='report any unexpected exceptions as problems' ) # parse arguments args = parser.parse_args() # sanity check arguments if not os.path.isfile(args.file): print '%s is not a file' % args.file sys.exit(1) with open(args.file, 'r') as f: # set up a csv reader for the data data = csv.reader(f, delimiter='\t') # create a validator validator = create_validator() # validate the data from the csv reader # N.B., validate() returns a list of problems; # if you expect a large number of problems, use ivalidate() instead # of validate(), but bear in mind that ivalidate() returns an iterator # so there is no len() problems = validator.validate(data, summarize=args.summarize, report_unexpected_exceptions=args.report_unexpected_exceptions, context={'file': args.file}) # write problems to stdout as restructured text write_problems(problems, sys.stdout, summarize=args.summarize, limit=args.limit) # decide how to exit if problems: # will not work with ivalidate() because it returns an iterator sys.exit(1) else: sys.exit(0)