def validate_csvreader(csvreader, filename):
count = 0
try:
# csvreader doesn't seem to generate any errors ever :-(
# csv module allows entire lines of json/xml/yaml to go in as a single field
# Adding some invalidations manually
for field_list in csvreader:
# list of fields with no separator information
log.debug("line: %s", field_list)
# make it fail if there is only a single field on any line
if len(field_list) < 3:
log.error("less than 3 fields detected, aborting conversion of file '%s'", filename)
return None
# extra protection along the same lines as anti-json:
# the first char of field should be alphanumeric, not syntax
# however instead of isAlnum allow quotes for quoted CSVs to pass validation
if not isChars(field_list[0][0], 'A-Za-z0-9"'):
log.error('non-alphanumeric / quote opening character detected in CSV')
return None
count += 1
except csv.Error as _:
log.warning('file %s, line %s: %s', filename, csvreader.line_num, _)
return None
if count == 0:
log.error('zero lines detected, blank input is not valid CSV')
return None
return csvreader
def process_line(self, line):
char = ''
if not line:
return
if not self.get_opt('no_comment'):
char = ' '
# preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it
line = line.strip()
if isChars(line[0], '#'):
char = line[0]
line = line.lstrip(char)
elif len(line) > 1 and isChars(line[0:1], '/'):
char = '//'
line = line.lstrip(char)
line = line.strip()
side = int(max((self.get_opt('width') - len(line)) / 2, 0))
print(char + ' ' * side + line)
def process_line(self, line):
char = ''
if not line:
return
if not self.get_opt('no_comment'):
char = ' '
# preliminary strip() to be able to pick up # if it isn't the first char and their are spaces before it
line = line.strip()
if isChars(line[0], '#'):
char = line[0]
line = line.lstrip(char)
elif len(line) > 1 and isChars(line[0:1], '/'):
char = '//'
line = line.lstrip(char)
line = line.strip()
side = int(max((self.get_opt('width') - len(line)) / 2, 0))
print(char + ' ' * side + line)
def process_csv(self, filehandle):
csvreader = None
try:
if self.delimiter is not None:
try:
csvreader = csv.reader(filehandle,
delimiter=self.delimiter,
quotechar=self.quotechar)
except TypeError as _:
self.usage(_)
else:
# dialect = csv.excel
dialect = csv.Sniffer().sniff(filehandle.read(1024))
# this will raise an Error if invalid
dialect.strict = True
filehandle.seek(0)
csvreader = csv.reader(filehandle, dialect)
except csv.Error as _:
log.warning('file %s: %s', self.filename, _)
return False
count = 0
try:
# csvreader doesn't seem to generate any errors ever :-(
# csv module allows entire lines of json/xml/yaml to go in as a single field
# Adding some invalidations manually
for field_list in csvreader:
# list of fields with no separator information
# log.debug("line: %s", _)
# make it fail if there is only a single field on any line
if len(field_list) < 2:
return False
# it's letting JSON through :-/
if field_list[0] == '{':
return False
# extra protection along the same lines as anti-json:
# the first char of field should be alphanumeric, not syntax
# however instead of isAlnum allow quotes for quoted CSVs to pass validation
if not isChars(field_list[0][0], 'A-Za-z0-9\'"'):
return False
count += 1
except csv.Error as _:
log.warning('file %s, line %s: %s', self.filename,
csvreader.line_num, _)
return False
if count == 0:
log.debug('zero lines detected, blank input is not valid CSV')
return False
log.debug('%s CSV lines passed', count)
return True
def process_csv(self, filehandle):
csvreader = None
try:
if self.delimiter is not None:
try:
csvreader = csv.reader(filehandle,
delimiter=self.delimiter,
quotechar=self.quotechar)
except TypeError as _:
self.usage(_)
else:
# dialect = csv.excel
dialect = csv.Sniffer().sniff(filehandle.read(1024))
# this will raise an Error if invalid
dialect.strict = True
filehandle.seek(0)
csvreader = csv.reader(filehandle, dialect)
except csv.Error as _:
if self.verbose > 2:
print('file {}: {}'.format(self.filename, _))
return False
try:
# csvreader doesn't seem to generate any errors ever :-(
# csv module allows entire lines of json/xml/yaml to go in as a single field
# Adding some invalidations manually
for _ in csvreader:
# log.debug("line: %s" % _)
# make it fail if there is only a single field on any line
if len(_) < 2:
return False
# it's letting JSON through :-/
if _[0] == '{':
return False
# extra protection along the same lines as anti-json:
# the first char of field should be alphanumeric, not syntax
# however instead of isAlnum allow quotes for quoted CSVs to pass validation
if not isChars(_[0][0], 'A-Za-z0-9\'"'):
return False
except csv.Error as _:
if self.verbose > 2:
print('file {}, line {}: {}'.format(self.filename,
csvreader.line_num, _))
return False
return True
def process_csv(self, filehandle):
csvreader = None
try:
if self.delimiter is not None:
try:
csvreader = csv.reader(filehandle, delimiter=self.delimiter, quotechar=self.quotechar)
except TypeError as _:
self.usage(_)
else:
# dialect = csv.excel
dialect = csv.Sniffer().sniff(filehandle.read(1024))
# this will raise an Error if invalid
dialect.strict = True
filehandle.seek(0)
csvreader = csv.reader(filehandle, dialect)
except csv.Error as _:
if self.verbose > 2:
print('file {}: {}'.format(self.filename, _))
return False
try:
# csvreader doesn't seem to generate any errors ever :-(
# csv module allows entire lines of json/xml/yaml to go in as a single field
# Adding some invalidations manually
for _ in csvreader:
# log.debug("line: %s" % _)
# make it fail if there is only a single field on any line
if len(_) < 2:
return False
# it's letting JSON through :-/
if _[0] == '{':
return False
# extra protection along the same lines as anti-json:
# the first char of field should be alphanumeric, not syntax
# however instead of isAlnum allow quotes for quoted CSVs to pass validation
if not isChars(_[0][0], 'A-Za-z0-9\'"'):
return False
except csv.Error as _:
if self.verbose > 2:
print('file {}, line {}: {}'.format(self.filename, csvreader.line_num, _))
return False
return True
