def sync_vs_async(query: str): client = get_search_client() now = datetime.now() async_results = client.search_async(query, num_results=30) print(f"{len(async_results)} async results in {datetime.now() - now}") now = datetime.now() sync_results = client.search(query, num_results=30) print(f"{len(sync_results)} sync results in {datetime.now() - now}")
def execute_query_export_html(query: str, save_html=False): client = get_search_client() results = client.search_async(query) time_str = get_timestamp() for i, r in enumerate(results): print(r.url) if save_html: # Write Result filepath = f"output/manual_client_search/{time_str}_{i}.html" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'a') as f: f.write(r.content)
def compare_query_results_with_train_data(): """ Compare the resulting URL's from searched articles with the ground truth articles given in training data """ num_examples = 1000 # Limit # of examples so this runs faster bqg = get_query_generator() client = get_search_client() ids = [] claims = [] queries = [] training_urls = [] client_urls = [] shared_urls_for_claim = [] # TODO: Finish this off for idx, claim in train_data_generator("/Users/frankjia/Desktop/LeadersPrize/train/train.json"): if idx == num_examples: break print(idx) # Execute query q = bqg.get_query(claim) res = client.search(q) searched_urls = [] # Get URLs from the result for r in res.results: searched_urls.append(r.url) # Get URL's from training data training_article_urls = [] for item in claim.related_articles: training_article_urls.append(item.url) # Get shared items shared_urls = list(set(training_article_urls).intersection(searched_urls)) ids.append(claim.id) claims.append(claim.claim) queries.append(q) training_urls.append(training_article_urls) client_urls.append(searched_urls) shared_urls_for_claim.append(shared_urls) # Get stats num_claims_with_shared_articles = sum([1 for item in shared_urls_for_claim if item]) frac_shared = float(num_claims_with_shared_articles) / len(ids) print(f"{num_claims_with_shared_articles} claims searched with shared articles out of {len(ids)}: {frac_shared}") num_shared_articles = sum([len(item) for item in shared_urls_for_claim]) total_train_articles = sum([len(item) for item in training_urls]) print(f"{num_shared_articles} shared articles found in {total_train_articles} total: {float(num_shared_articles) / total_train_articles}")
def execute_queries_export_urls(): """ Output the queries as well as the resulting URL's from the search client Will also create the HTML files returned from the query """ num_examples = 30 # Limit # of examples so this runs faster bqg = get_query_generator() client = get_search_client() timestamp = get_timestamp() ids = [] original_claims = [] data = [] for idx, claim in enumerate(train_data_generator()): if idx == num_examples: break ids.append(claim.id) original_claims.append(claim.claim) q = bqg.get_query(claim) res = client.search(q) export_str = "" for i, r in enumerate(res.results): # Add to URLs export_str += f"{r.score}: {r.url} | " # Write Result filepath = f"output/basic_query_generator/query_html_{timestamp}/{claim.id}_{i}.html" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'a') as f: f.write(r.content) data.append(export_str) export_df = pd.DataFrame(data={ "id": ids, "original": original_claims, "results": data }) save_results(export_df, "basic_query_generator", "query_to_url", time_str=timestamp)
def main(): # Services query_generator = get_query_generator() client = get_search_client() html_preprocessor = get_html_preprocessor() text_preprocessor = get_text_preprocessor() # Create the appropriate relevance scorer relevance_scorer = get_infersent_relevance_scorer() if RELEVANCE_TYPE == INFERSENT_RELEVANCE_SCORER \ else get_word2vec_relevance_scorer() # Outputs ids = [] original_claims = [] processed_claims = [] queries = [] processed_sentences = [] true_labels = [] for idx, claim in enumerate(train_data_generator()): if idx == NUM_EXAMPLES: break print(idx) query = query_generator.get_query(claim) search_res = client.search(query) processed_claim = ' '.join( text_preprocessor.process( claim.claim).bert_sentences) # Preprocessed claim ids.append(claim.id) original_claims.append(claim.claim) true_labels.append(claim.label) queries.append(query) processed_claims.append(processed_claim) # Continue in case of error if search_res.error: processed_sentences.append(f"Error: {search_res.error}") continue # Create master list of sentences sentences = [] for article in search_res.results: html_processed = html_preprocessor.process(article.content).text text_processed = text_preprocessor.process(html_processed) sentences += text_processed.bert_sentences # Run relevance scores if RELEVANCE_TYPE == INFERSENT_RELEVANCE_SCORER: relevances = get_infersent_relevances(claim.claim, sentences, relevance_scorer) else: relevances = get_word2vec_relevances(claim.claim, sentences, relevance_scorer) # Combine the two results processed_sentences_with_relevance = list(zip(relevances, sentences)) # Construct final string processed_sentences_with_relevance.sort(key=lambda item: item[0], reverse=True) process_result = "" for rel, sent in processed_sentences_with_relevance: if len(process_result) > 10000: # Some basic truncation to limit file size break process_result += f"|SEP| {rel}: {sent} \n" processed_sentences.append(process_result) # Export the result, with relevance scores and the processed text export_df = pd.DataFrame( data={ "id": ids, "label": true_labels, "original": original_claims, "query": queries, "processed_claim": processed_claims, "processed_sentences": processed_sentences }) save_results(export_df, "sentence_relevance_scorer", f"claim_to_{RELEVANCE_TYPE}_relevance")
from experiments.util.experiment_util import get_search_client import json search_client = get_search_client() root_folder_path = "/Users/frankjia/Desktop/LeadersPrize/train_phase2_val" train_json_filepath = "train_val_phase2.json" new_train_json_path = "train.json" with open(f"{root_folder_path}/{train_json_filepath}", "r") as f: train_json = json.load(f) new_train_json = [] article_key = 0 for item in train_json: print(item["id"]) new_rel_articles = {} given_rel_articles = item["related_articles"] for _, article_url in given_rel_articles.items(): results = search_client.search(article_url, num_results=30) url_matched_result = None for result in results: if result.url == article_url: url_matched_result = result break if url_matched_result: # Write the output article_filepath = f"train_articles/{article_key}.html" with open(f"{root_folder_path}/{article_filepath}", "w") as f: f.write(url_matched_result.content) article_key += 1 # add to new rel articles new_rel_articles[article_filepath] = url_matched_result.url
def test_document_relevance_scorer(): """ This runs the pipeline up to retrieving articles and ranking them by relevance. Determines, if there is shared articles between what is retrieved and what is given as related_articles, the ranking of the related article """ num_examples = 1000 # Limit # of examples so this runs faster bqg = get_query_generator() client = get_search_client() html_preprocessor = get_html_preprocessor() text_prepreprocessor = get_text_preprocessor() article_relevance_scorer = get_lsa_relevance_scorer() total_searched = 0 average_rankings = [] for idx, claim in train_data_generator( "/Users/frankjia/Desktop/LeadersPrize/train/train.json"): if idx == num_examples: break print(idx) # Execute query q = bqg.get_query(claim) searched_articles = client.search(q).results # Process articles from raw HTML to parsed text pipeline_articles: List[PipelineArticle] = [] for raw_article in searched_articles: if raw_article and raw_article.content: pipeline_article = PipelineArticle(raw_article) # Extract data from HTML html_process_result = html_preprocessor.process( raw_article.content) pipeline_article.html_attributes = html_process_result.html_atts pipeline_article.raw_body_text = html_process_result.text pipeline_articles.append(pipeline_article) # Get Article Relevance preprocessed_claim_sentences = text_prepreprocessor.process( claim.claim + " " + claim.claimant).sentences preprocessed_claim = claim.claim + " " + claim.claimant if preprocessed_claim_sentences: preprocessed_claim = preprocessed_claim_sentences[0] pipeline_article_texts: List[str] = [ p.raw_body_text for p in pipeline_articles ] article_relevances = article_relevance_scorer.analyze( preprocessed_claim, pipeline_article_texts) for article_relevance, pipeline_article in zip(article_relevances, pipeline_articles): # Sometimes we get nan from numpy operations pipeline_article.relevance = article_relevance if math.isfinite( article_relevance) else 0 # Based on article relevance, only consider the top relevances pipeline_articles.sort(key=lambda article: article.relevance, reverse=True) sorted_urls = [article.url for article in pipeline_articles] claim_urls = [article.url for article in claim.related_articles] common_urls = list(set(sorted_urls).intersection(claim_urls)) total_searched += 1 if common_urls: # Determine index of shared url in the sorted urls index_sum = 0 for url in common_urls: index_sum += sorted_urls.index(url) average_rankings.append(float(index_sum) / len(common_urls)) print("RESULTS") print(total_searched) print(len(average_rankings)) print(float(sum(average_rankings)) / len(average_rankings))
def main(): """ For each claim, run query generator and rank the results by relevance using the LSA document relevance scorer """ # Services query_generator = get_query_generator() client = get_search_client() html_preprocessor = get_html_preprocessor() relevance_scorer = get_lsa_relevance_scorer() # Outputs ids = [] original_claims = [] ranked_articles_for_claims = [] for idx, claim in train_data_generator( "/Users/frankjia/Desktop/LeadersPrize/train/train.json"): if idx == NUM_EXAMPLES: break print(idx) query = query_generator.get_query(claim) search_res = client.search(query) ids.append(claim.id) original_claims.append(claim.claim) # Create master list of sentences article_texts = [] article_urls = [] for article in search_res.results: # Write the article for future checking url_filename = re.sub( r"([/:.])+", "_", article.url) # Create a save-friendly filename filepath = f"output/{claim.id}/{url_filename}.html" os.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'a') as f: f.write(article.content) # Process the articles html_processed_text = html_preprocessor.process( article.content).text article_urls.append(article.url) article_texts.append(html_processed_text) # Both claim and article_texts are unpreprocessed - LSA class currently does the preprocessing relevances = relevance_scorer.analyze(claim.claim, article_texts) print(relevances) articles_with_relevances = list(zip(article_urls, relevances)) articles_with_relevances.sort(key=lambda x: x[1], reverse=True) # Create an export string with the URL and the relevance: article_rank_result = "" for url, rel in articles_with_relevances: article_rank_result += f"( {rel}: {url} )" ranked_articles_for_claims.append(article_rank_result) # Export the result, with relevance scores and the processed text export_df = pd.DataFrame( data={ "id": ids, "claim": original_claims, "ranked_articles": ranked_articles_for_claims }) save_results(export_df, "document_relevance_scorer", "claim_to_ranked_articles")