def read_encoded(path, version, callback, column_headers, encoding): # Coerce to a list if not hasattr(version, 'extend'): version = [version] with io.open(path, 'r', encoding=encoding) as csvfile: plainreader = unicode_csv_reader(csvfile, **dialect) for row in plainreader: column_type = row[0] if column_type == 'RH': if int(row[4]) not in version: raise RuntimeError("This file uses an unexpected format revision: {version}".format(version=row[4])) elif column_type == 'FH': pass elif column_type == 'SH': start_date, end_date = row[1:3] log.info("Report file covers date range {start} to {end}".format(start=start_date, end=end_date)) elif column_type == 'CH': column_headers = ['Column Type'] + row[1:] elif column_type == 'SB': record = dict(zip(column_headers, row)) try: callback(record) except: FailMailer.mail('BAD_AUDIT_LINE', data=record, print_exception=True) elif column_type in ('SF', 'SC', 'RF', 'RC', 'FF'): pass else: raise RuntimeError("Unknown column type: {type}".format(type=column_type))
def __loadFile(self, filename): file = codecs.open(filename, 'r', 'utf-8') #reader = csv.reader(file) reader = unicode_csv_reader(file) self.contents = [] for ii, entry in enumerate(reader): if ii == 0: self.__mapColumns(entry) continue line = {} for jj in range(len(entry)): line[self.columns_to_names[jj]] = entry[jj] self.contents.append(line) file.close()
def read_encoded(path, version, callback, column_headers, encoding): # Coerce to a list if not hasattr(version, 'extend'): version = [version] with io.open(path, 'r', encoding=encoding) as csvfile: plainreader = unicode_csv_reader(csvfile, **dialect) rownum = 1 for row in plainreader: column_type = row[0] if column_type == 'RH': if int(row[4]) not in version: raise RuntimeError("This file uses an unexpected format revision: {version}".format(version=row[4])) elif column_type == 'FH': pass elif column_type == 'SH': start_date, end_date = row[1:3] log.info("Report file covers date range {start} to {end}".format(start=start_date, end=end_date)) elif column_type == 'CH': column_headers = ['Column Type'] + row[1:] elif column_type == 'SB': record = dict(zip(column_headers, row)) try: callback(record) except Exception: logme = { 'file': os.path.basename(path), 'row': rownum } for identifier in ['Transaction ID', 'Invoice ID', 'PayPal Reference ID', 'Subscription ID']: if identifier in record: logme[identifier] = record[identifier] FailMailer.mail('BAD_AUDIT_LINE', data=logme, print_exception=True) elif column_type in ('SF', 'SC', 'RF', 'RC', 'FF'): pass else: raise RuntimeError("Unknown column type: {type}".format(type=column_type)) rownum = rownum + 1
def read_encoded(path, version, callback, column_headers, encoding): # Coerce to a list if not hasattr(version, 'extend'): version = [version] with io.open(path, 'r', encoding=encoding) as csvfile: plainreader = unicode_csv_reader(csvfile, **dialect) for row in plainreader: column_type = row[0] if column_type == 'RH': if int(row[4]) not in version: raise RuntimeError( "This file uses an unexpected format revision: {version}" .format(version=row[4])) elif column_type == 'FH': pass elif column_type == 'SH': start_date, end_date = row[1:3] log.info( "Report file covers date range {start} to {end}".format( start=start_date, end=end_date)) elif column_type == 'CH': column_headers = ['Column Type'] + row[1:] elif column_type == 'SB': record = dict(zip(column_headers, row)) try: callback(record) except: FailMailer.mail('BAD_AUDIT_LINE', data=record, print_exception=True) elif column_type in ('SF', 'SC', 'RF', 'RC', 'FF'): pass else: raise RuntimeError( "Unknown column type: {type}".format(type=column_type))