def store_raw_response(self, source_details, response_text): """Store raw response from API provider using source_details""" cls = sql_convenience.CLASS_NOT_INVESTIGATED json_response = json.dumps(response_text) sql_convenience.insert_api_response(source_details[b'tweet_id'], source_details[b'tweet_text'], json_response, cls, config.db_conn, self.destination_table)
def store_raw_response(self, source_details, response_text): """Store raw response from API provider using source_details""" cls = sql_convenience.CLASS_NOT_INVESTIGATED json_response = json.dumps(response_text) sql_convenience.insert_api_response( source_details[b"tweet_id"], source_details[b"tweet_text"], json_response, cls, config.db_conn, self.destination_table, )
def copy_data_to_subsets(cls, source_table, nbr_testtrain, testtrain_table, nbr_validation, validation_table, drop, config): cursor = config.db_conn.cursor() sql = "SELECT * FROM {} WHERE class=={}".format(source_table, cls) cursor.execute(sql) rows = cursor.fetchall() random.shuffle(rows) rows_validation = rows[:nbr_validation] rows_traintest = rows[nbr_validation:nbr_validation + nbr_testtrain] #print(rows_traintest[0][b'tweet_id']) #print(rows_validation[0][b'tweet_id']) # move this n using sql to a new table sql_convenience.create_results_table(config.db_conn, testtrain_table, force_drop_table=drop) sql_convenience.create_results_table(config.db_conn, validation_table, force_drop_table=drop) for row in rows_traintest: sql_convenience.insert_api_response(row[b'tweet_id'], row[b'tweet_text'], "", cls, config.db_conn, testtrain_table) for row in rows_validation: sql_convenience.insert_api_response(row[b'tweet_id'], row[b'tweet_text'], "", cls, config.db_conn, validation_table)