save_batch(batch) batch = [] job.end() seek_confirmation() #exit() for model_name in ["logistic_regression", "multinomial_nb"]: storage = ModelStorage(dirpath=f"nlp_v2/models/best/{model_name}") tv = storage.load_vectorizer() clf = storage.load_model() print(f"DESTROY PREDICTIONS TABLE? ({model_name})") seek_confirmation() bq_service.nlp_v2_destructively_migrate_predictions_table(model_name) predictions_table = bq_service.nlp_v2_get_predictions_table(model_name) # API call. cache it here once. job.start() for chunk_df in read_csv(CSV_FILEPATH, chunksize=BATCH_SIZE): # FYI: this will include the last chunk even if it is not a full batch status_ids = chunk_df["status_id"].tolist() status_texts = chunk_df["status_text"].tolist() preds = clf.predict(tv.transform(status_texts)) batch = [{"status_id": status_id, "prediction": pred} for status_id, pred in zip(status_ids, preds)] bq_service.insert_records_in_batches(predictions_table, batch) job.counter += len(chunk_df) job.progress_report()
from app.job import Job from app.bq_service import BigQueryService from app.retweet_graphs_v2.k_days.generator import DateRangeGenerator BATCH_SIZE = int( os.getenv("BATCH_SIZE", default=25000) ) # the max number of processed users to store in BQ at once (with a single insert API call) if __name__ == "__main__": bq_service = BigQueryService() job = Job() print(f"DESTROY PREDICTIONS TABLE? (BERT)") seek_confirmation() bq_service.nlp_v2_destructively_migrate_predictions_table("bert") predictions_table = bq_service.nlp_v2_get_predictions_table("bert") job.start() for dr in DateRangeGenerator(start_date="2019-12-20", k_days=1, n_periods=58).date_ranges: print(dr.start_date) csv_filepath = os.path.join(DATA_DIR, "daily_active_edge_friend_graphs_v5", dr.start_date, "tweets_BERT_Impeachment_800KTweets.csv") #df = read_csv(csv_filepath, usecols=["status_id", "text", "logit_0", "logit_1", "opinion_tweet"], nrows=100) #print(df.head())