def validate_csvreader(csvreader, filename): count = 0 try: # csvreader doesn't seem to generate any errors ever :-( # csv module allows entire lines of json/xml/yaml to go in as a single field # Adding some invalidations manually for field_list in csvreader: # list of fields with no separator information log.debug("line: %s", field_list) # make it fail if there is only a single field on any line if len(field_list) < 3: log.error("less than 3 fields detected, aborting conversion of file '%s'", filename) return None # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation if not isChars(field_list[0][0], 'A-Za-z0-9"'): log.error('non-alphanumeric / quote opening character detected in CSV') return None count += 1 except csv.Error as _: log.warning('file %s, line %s: %s', filename, csvreader.line_num, _) return None if count == 0: log.error('zero lines detected, blank input is not valid CSV') return None return csvreader
def process_line(self, line): char = '' if not line: return if not self.get_opt('no_comment'): char = ' ' # preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it line = line.strip() if isChars(line[0], '#'): char = line[0] line = line.lstrip(char) elif len(line) > 1 and isChars(line[0:1], '/'): char = '//' line = line.lstrip(char) line = line.strip() side = int(max((self.get_opt('width') - len(line)) / 2, 0)) print(char + ' ' * side + line)
def process_csv(self, filehandle): csvreader = None try: if self.delimiter is not None: try: csvreader = csv.reader(filehandle, delimiter=self.delimiter, quotechar=self.quotechar) except TypeError as _: self.usage(_) else: # dialect = csv.excel dialect = csv.Sniffer().sniff(filehandle.read(1024)) # this will raise an Error if invalid dialect.strict = True filehandle.seek(0) csvreader = csv.reader(filehandle, dialect) except csv.Error as _: log.warning('file %s: %s', self.filename, _) return False count = 0 try: # csvreader doesn't seem to generate any errors ever :-( # csv module allows entire lines of json/xml/yaml to go in as a single field # Adding some invalidations manually for field_list in csvreader: # list of fields with no separator information # log.debug("line: %s", _) # make it fail if there is only a single field on any line if len(field_list) < 2: return False # it's letting JSON through :-/ if field_list[0] == '{': return False # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation if not isChars(field_list[0][0], 'A-Za-z0-9\'"'): return False count += 1 except csv.Error as _: log.warning('file %s, line %s: %s', self.filename, csvreader.line_num, _) return False if count == 0: log.debug('zero lines detected, blank input is not valid CSV') return False log.debug('%s CSV lines passed', count) return True
def process_csv(self, filehandle): csvreader = None try: if self.delimiter is not None: try: csvreader = csv.reader(filehandle, delimiter=self.delimiter, quotechar=self.quotechar) except TypeError as _: self.usage(_) else: # dialect = csv.excel dialect = csv.Sniffer().sniff(filehandle.read(1024)) # this will raise an Error if invalid dialect.strict = True filehandle.seek(0) csvreader = csv.reader(filehandle, dialect) except csv.Error as _: if self.verbose > 2: print('file {}: {}'.format(self.filename, _)) return False try: # csvreader doesn't seem to generate any errors ever :-( # csv module allows entire lines of json/xml/yaml to go in as a single field # Adding some invalidations manually for _ in csvreader: # log.debug("line: %s" % _) # make it fail if there is only a single field on any line if len(_) < 2: return False # it's letting JSON through :-/ if _[0] == '{': return False # extra protection along the same lines as anti-json: # the first char of field should be alphanumeric, not syntax # however instead of isAlnum allow quotes for quoted CSVs to pass validation if not isChars(_[0][0], 'A-Za-z0-9\'"'): return False except csv.Error as _: if self.verbose > 2: print('file {}, line {}: {}'.format(self.filename, csvreader.line_num, _)) return False return True