Body=json.dumps(out), ContentType='application/json; charset=utf-8') def generate_json(self, rows): s3 = boto3.resource('s3') out = {'ads': [dict(r) for r in SAVER.get_rows()]} s3.Bucket(conf.S3_BUCKET).put_object( Key='out.json', Body=json.dumps(out), ContentType='application/json; charset=utf-8') if __name__ == '__main__': OUT_DIR = 'ads' if len(sys.argv) > 1: OUT_DIR = sys.argv[1] CRAWLER = Crawler(OUT_DIR) RUN_ID, ADS = CRAWLER.crawl() CRAWLER.upload_to_s3(RUN_ID, ADS) SAVER = Saver(cred_file='client_secret.json') SAVER.open_workbook('Sleeping Giants Data') for AD in ADS: SAVER.insert_row( [AD['orig'], AD['ad'], AD['twitter_account'], AD['curr_url']]) CRAWLER.generate_json(SAVER.get_rows())