def handle_sheet( self, year, filename ):
        dataset = Dataset('raw', self.MUNI, year)
        reader = csv.reader(file(filename, 'rb'))

        for line in reader:
            new_line = {}

            line_fields = [self.fields[index](line[index])
                                for index in self.fields.keys()]

            # check validity of line and write valid lines to DB
            fields_are_valid = [field.is_valid() for field in line_fields]

            if all(fields_are_valid):
                for field in line_fields:
                    # process fields
                    new_line[field.name] = field.process()

                # insert line data to DB
                self.print_str("%s : %s" %(line['code'], line['amount']))
                self.print_str(new_line)
                dataset.insert(new_line)
            else:
                invalid_fields = [' : '.join([field.name, field.value, field.error]) 
                                  for field in line_fields if field.is_valid()]
                self.logger.info('invalid fields: %s', ' '.join(invalid_fields))
        dataset.close()
Exemple #2
0
    def handle_sheet( self, year, filename ):
        # TODO: do this more generic than this!
        dataset = Dataset('raw', self.MUNI, year)
        reader = csv.reader(file(filename, 'rb'))

        if year in self.years:
            fields = self.years[year]
        else:
            fields = self.fields

        for line in reader:
            new_line = {}

            line_fields = [fields[index](line[index])
                                for index in fields]

            # check validity of line and write valid lines to DB
            fields_are_valid = [field.is_valid() for field in line_fields]

            if all(fields_are_valid):
                for field in line_fields:
                    # process fields
                    new_line[field.name] = field.process()

                # insert line data to DB
                self.print_str(new_line)
                dataset.insert(new_line)
            else:
                invalid_fields = [' : '.join([field.name, field.value, field.error()]) 
                                  for field in line_fields if not field.is_valid()]
                #self.logger.info('invalid fields: %s', ' '.join(invalid_fields))
                print 'invalid fields: %s' %(' '.join(invalid_fields),)
        dataset.close()
Exemple #3
0
    def handle_sheet(self, year, filename):
         dataset = Dataset('raw', self.MUNI, year)
         reader = csv.DictReader(file(filename, 'rb'), self.fields)

         for line in reader:
            self.print_str("%s : %s" %(line['code'], line['amount']))
            if (line['name'] != '' and line['amount'].isdigit()):
                new_line  = {'name':line['name'], 'amount':line['amount'], 'code':line['code'] }
                dataset.insert(new_line)
         dataset.close()