def rows(self): sanitised_streams = self.input()["sanitised_streams"] with sanitised_streams.open('r') as in_file: for line in in_file: record = json_loads(line.strip('\n')) logger.info("Looking at %s with id %s" % (record["record_type"], record["report_id"])) if record["record_type"] == "header": logger.info("Found header") yield self.format_record(record)
def rows(self): with self.input().open('r') as in_file: for line in in_file: record = json_loads(line.decode('utf-8', 'ignore').strip('\n')) logger.info("Adding to DB %s" % (record["report_id"])) yield self.serialize(record)