def rows(self):
sanitised_streams = self.input()["sanitised_streams"]
with sanitised_streams.open('r') as in_file:
for line in in_file:
record = json_loads(line.strip('\n'))
logger.info("Looking at %s with id %s" % (record["record_type"], record["report_id"]))
if record["record_type"] == "header":
logger.info("Found header")
yield self.format_record(record)
def rows(self):
with self.input().open('r') as in_file:
for line in in_file:
record = json_loads(line.decode('utf-8', 'ignore').strip('\n'))
logger.info("Adding to DB %s" % (record["report_id"]))
yield self.serialize(record)
