def handle_sheet( self, year, filename ):
dataset = Dataset('raw', self.MUNI, year)
reader = csv.reader(file(filename, 'rb'))
for line in reader:
new_line = {}
line_fields = [self.fields[index](line[index])
for index in self.fields.keys()]
# check validity of line and write valid lines to DB
fields_are_valid = [field.is_valid() for field in line_fields]
if all(fields_are_valid):
for field in line_fields:
# process fields
new_line[field.name] = field.process()
# insert line data to DB
self.print_str("%s : %s" %(line['code'], line['amount']))
self.print_str(new_line)
dataset.insert(new_line)
else:
invalid_fields = [' : '.join([field.name, field.value, field.error])
for field in line_fields if field.is_valid()]
self.logger.info('invalid fields: %s', ' '.join(invalid_fields))
dataset.close()
def handle_sheet( self, year, filename ):
# TODO: do this more generic than this!
dataset = Dataset('raw', self.MUNI, year)
reader = csv.reader(file(filename, 'rb'))
if year in self.years:
fields = self.years[year]
else:
fields = self.fields
for line in reader:
new_line = {}
line_fields = [fields[index](line[index])
for index in fields]
# check validity of line and write valid lines to DB
fields_are_valid = [field.is_valid() for field in line_fields]
if all(fields_are_valid):
for field in line_fields:
# process fields
new_line[field.name] = field.process()
# insert line data to DB
self.print_str(new_line)
dataset.insert(new_line)
else:
invalid_fields = [' : '.join([field.name, field.value, field.error()])
for field in line_fields if not field.is_valid()]
#self.logger.info('invalid fields: %s', ' '.join(invalid_fields))
print 'invalid fields: %s' %(' '.join(invalid_fields),)
dataset.close()
def handle_sheet(self, year, filename):
dataset = Dataset('raw', self.MUNI, year)
reader = csv.DictReader(file(filename, 'rb'), self.fields)
for line in reader:
self.print_str("%s : %s" %(line['code'], line['amount']))
if (line['name'] != '' and line['amount'].isdigit()):
new_line = {'name':line['name'], 'amount':line['amount'], 'code':line['code'] }
dataset.insert(new_line)
dataset.close()