def create_and_export_queries():
"""
Output the generated queries only
"""
num_examples = 100
bqg = get_query_generator()
truth_tup_extractor = get_truth_tuple_extractor()
ids = []
original_claims = []
processed = []
for idx, claim in enumerate(train_data_generator()):
if idx == num_examples:
break
ids.append(claim.id)
original_claims.append(claim.claim)
claim_truth_tuples = truth_tup_extractor.extract(claim.claimant + " " +
claim.claim)
processed.append(bqg.get_query(claim, truth_tuples=claim_truth_tuples))
export_df = pd.DataFrame(data={
"id": ids,
"original": original_claims,
"queries": processed
})
save_results(export_df, "basic_query_generator", "queries")
def compare_query_results_with_train_data():
"""
Compare the resulting URL's from searched articles with the ground truth articles given in training data
"""
num_examples = 1000 # Limit # of examples so this runs faster
bqg = get_query_generator()
client = get_search_client()
ids = []
claims = []
queries = []
training_urls = []
client_urls = []
shared_urls_for_claim = []
# TODO: Finish this off
for idx, claim in train_data_generator("/Users/frankjia/Desktop/LeadersPrize/train/train.json"):
if idx == num_examples:
break
print(idx)
# Execute query
q = bqg.get_query(claim)
res = client.search(q)
searched_urls = []
# Get URLs from the result
for r in res.results:
searched_urls.append(r.url)
# Get URL's from training data
training_article_urls = []
for item in claim.related_articles:
training_article_urls.append(item.url)
# Get shared items
shared_urls = list(set(training_article_urls).intersection(searched_urls))
ids.append(claim.id)
claims.append(claim.claim)
queries.append(q)
training_urls.append(training_article_urls)
client_urls.append(searched_urls)
shared_urls_for_claim.append(shared_urls)
# Get stats
num_claims_with_shared_articles = sum([1 for item in shared_urls_for_claim if item])
frac_shared = float(num_claims_with_shared_articles) / len(ids)
print(f"{num_claims_with_shared_articles} claims searched with shared articles out of {len(ids)}: {frac_shared}")
num_shared_articles = sum([len(item) for item in shared_urls_for_claim])
total_train_articles = sum([len(item) for item in training_urls])
print(f"{num_shared_articles} shared articles found in {total_train_articles} total: {float(num_shared_articles) / total_train_articles}")
def main():
html_preprocessor = get_html_preprocessor()
error_files = []
for idx, claim in train_data_generator(TRAIN_DATA_PATH + "train.json"):
if idx < PROCESS_RANGE.start:
continue
elif idx >= PROCESS_RANGE.stop:
break
# Add the articles if we're not retrieving from search client
for related_article in claim.related_articles:
article_html = get_train_article(TRAIN_DATA_PATH, related_article.filepath)
if not html_preprocessor.process(article_html).text:
print(related_article.filepath)
error_files.append(related_article.filepath)
output = '\n'.join(error_files)
with open('../output/html_processor/html_unprocessable_files.txt', 'a') as f:
f.write(output)
def execute_queries_export_urls():
"""
Output the queries as well as the resulting URL's from the search client
Will also create the HTML files returned from the query
"""
num_examples = 30 # Limit # of examples so this runs faster
bqg = get_query_generator()
client = get_search_client()
timestamp = get_timestamp()
ids = []
original_claims = []
data = []
for idx, claim in enumerate(train_data_generator()):
if idx == num_examples:
break
ids.append(claim.id)
original_claims.append(claim.claim)
q = bqg.get_query(claim)
res = client.search(q)
export_str = ""
for i, r in enumerate(res.results):
# Add to URLs
export_str += f"{r.score}: {r.url} | "
# Write Result
filepath = f"output/basic_query_generator/query_html_{timestamp}/{claim.id}_{i}.html"
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, 'a') as f:
f.write(r.content)
data.append(export_str)
export_df = pd.DataFrame(data={
"id": ids,
"original": original_claims,
"results": data
})
save_results(export_df,
"basic_query_generator",
"query_to_url",
time_str=timestamp)
def main():
# Services
query_generator = get_query_generator()
client = get_search_client()
html_preprocessor = get_html_preprocessor()
text_preprocessor = get_text_preprocessor()
# Create the appropriate relevance scorer
relevance_scorer = get_infersent_relevance_scorer() if RELEVANCE_TYPE == INFERSENT_RELEVANCE_SCORER \
else get_word2vec_relevance_scorer()
# Outputs
ids = []
original_claims = []
processed_claims = []
queries = []
processed_sentences = []
true_labels = []
for idx, claim in enumerate(train_data_generator()):
if idx == NUM_EXAMPLES:
break
print(idx)
query = query_generator.get_query(claim)
search_res = client.search(query)
processed_claim = ' '.join(
text_preprocessor.process(
claim.claim).bert_sentences) # Preprocessed claim
ids.append(claim.id)
original_claims.append(claim.claim)
true_labels.append(claim.label)
queries.append(query)
processed_claims.append(processed_claim)
# Continue in case of error
if search_res.error:
processed_sentences.append(f"Error: {search_res.error}")
continue
# Create master list of sentences
sentences = []
for article in search_res.results:
html_processed = html_preprocessor.process(article.content).text
text_processed = text_preprocessor.process(html_processed)
sentences += text_processed.bert_sentences
# Run relevance scores
if RELEVANCE_TYPE == INFERSENT_RELEVANCE_SCORER:
relevances = get_infersent_relevances(claim.claim, sentences,
relevance_scorer)
else:
relevances = get_word2vec_relevances(claim.claim, sentences,
relevance_scorer)
# Combine the two results
processed_sentences_with_relevance = list(zip(relevances, sentences))
# Construct final string
processed_sentences_with_relevance.sort(key=lambda item: item[0],
reverse=True)
process_result = ""
for rel, sent in processed_sentences_with_relevance:
if len(process_result) > 10000:
# Some basic truncation to limit file size
break
process_result += f"|SEP| {rel}: {sent} \n"
processed_sentences.append(process_result)
# Export the result, with relevance scores and the processed text
export_df = pd.DataFrame(
data={
"id": ids,
"label": true_labels,
"original": original_claims,
"query": queries,
"processed_claim": processed_claims,
"processed_sentences": processed_sentences
})
save_results(export_df, "sentence_relevance_scorer",
f"claim_to_{RELEVANCE_TYPE}_relevance")
def test_document_relevance_scorer():
"""
This runs the pipeline up to retrieving articles and ranking them by relevance.
Determines, if there is shared articles between what is retrieved and what is given as related_articles,
the ranking of the related article
"""
num_examples = 1000 # Limit # of examples so this runs faster
bqg = get_query_generator()
client = get_search_client()
html_preprocessor = get_html_preprocessor()
text_prepreprocessor = get_text_preprocessor()
article_relevance_scorer = get_lsa_relevance_scorer()
total_searched = 0
average_rankings = []
for idx, claim in train_data_generator(
"/Users/frankjia/Desktop/LeadersPrize/train/train.json"):
if idx == num_examples:
break
print(idx)
# Execute query
q = bqg.get_query(claim)
searched_articles = client.search(q).results
# Process articles from raw HTML to parsed text
pipeline_articles: List[PipelineArticle] = []
for raw_article in searched_articles:
if raw_article and raw_article.content:
pipeline_article = PipelineArticle(raw_article)
# Extract data from HTML
html_process_result = html_preprocessor.process(
raw_article.content)
pipeline_article.html_attributes = html_process_result.html_atts
pipeline_article.raw_body_text = html_process_result.text
pipeline_articles.append(pipeline_article)
# Get Article Relevance
preprocessed_claim_sentences = text_prepreprocessor.process(
claim.claim + " " + claim.claimant).sentences
preprocessed_claim = claim.claim + " " + claim.claimant
if preprocessed_claim_sentences:
preprocessed_claim = preprocessed_claim_sentences[0]
pipeline_article_texts: List[str] = [
p.raw_body_text for p in pipeline_articles
]
article_relevances = article_relevance_scorer.analyze(
preprocessed_claim, pipeline_article_texts)
for article_relevance, pipeline_article in zip(article_relevances,
pipeline_articles):
# Sometimes we get nan from numpy operations
pipeline_article.relevance = article_relevance if math.isfinite(
article_relevance) else 0
# Based on article relevance, only consider the top relevances
pipeline_articles.sort(key=lambda article: article.relevance,
reverse=True)
sorted_urls = [article.url for article in pipeline_articles]
claim_urls = [article.url for article in claim.related_articles]
common_urls = list(set(sorted_urls).intersection(claim_urls))
total_searched += 1
if common_urls:
# Determine index of shared url in the sorted urls
index_sum = 0
for url in common_urls:
index_sum += sorted_urls.index(url)
average_rankings.append(float(index_sum) / len(common_urls))
print("RESULTS")
print(total_searched)
print(len(average_rankings))
print(float(sum(average_rankings)) / len(average_rankings))
def main():
"""
For each claim, run query generator and rank the results by relevance using the LSA document relevance scorer
"""
# Services
query_generator = get_query_generator()
client = get_search_client()
html_preprocessor = get_html_preprocessor()
relevance_scorer = get_lsa_relevance_scorer()
# Outputs
ids = []
original_claims = []
ranked_articles_for_claims = []
for idx, claim in train_data_generator(
"/Users/frankjia/Desktop/LeadersPrize/train/train.json"):
if idx == NUM_EXAMPLES:
break
print(idx)
query = query_generator.get_query(claim)
search_res = client.search(query)
ids.append(claim.id)
original_claims.append(claim.claim)
# Create master list of sentences
article_texts = []
article_urls = []
for article in search_res.results:
# Write the article for future checking
url_filename = re.sub(
r"([/:.])+", "_",
article.url) # Create a save-friendly filename
filepath = f"output/{claim.id}/{url_filename}.html"
os.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, 'a') as f:
f.write(article.content)
# Process the articles
html_processed_text = html_preprocessor.process(
article.content).text
article_urls.append(article.url)
article_texts.append(html_processed_text)
# Both claim and article_texts are unpreprocessed - LSA class currently does the preprocessing
relevances = relevance_scorer.analyze(claim.claim, article_texts)
print(relevances)
articles_with_relevances = list(zip(article_urls, relevances))
articles_with_relevances.sort(key=lambda x: x[1], reverse=True)
# Create an export string with the URL and the relevance:
article_rank_result = ""
for url, rel in articles_with_relevances:
article_rank_result += f"( {rel}: {url} )"
ranked_articles_for_claims.append(article_rank_result)
# Export the result, with relevance scores and the processed text
export_df = pd.DataFrame(
data={
"id": ids,
"claim": original_claims,
"ranked_articles": ranked_articles_for_claims
})
save_results(export_df, "document_relevance_scorer",
"claim_to_ranked_articles")
def test_pipeline(process_range: range, config: Dict, train_data_path: str):
raw_claims: List[LeadersPrizeClaim] = []
raw_claim_dicts: List[dict] = []
init_articles = not config.get(PipelineConfigKeys.RETRIEVE_ARTICLES, True)
if init_articles:
print(
"Reading articles from training data. Will not call search client")
for idx, claim_dict, claim in train_data_generator(
train_data_path + "trial_combined_data_long.json"):
if idx < process_range.start:
continue
elif idx >= process_range.stop:
break
# Add the articles if we're not retrieving from search client
if init_articles:
articles: List[SearchQueryResult] = []
for related_article in claim.related_articles:
article_html = get_train_article(train_data_path,
related_article.filepath)
articles.append(
SearchQueryResult(content=article_html,
url=related_article.url))
claim.mock_search_results = articles
raw_claims.append(claim)
raw_claim_dicts.append(claim_dict)
start_time = datetime.now()
# Create pipeline
pipeline = LeadersPrizePipeline(config)
# Run the prediction
results = pipeline.predict(raw_claims)
print(f"{len(results)} processed in {datetime.now() - start_time}")
# Export results
claims = []
labels = []
reasoner_inputs = []
pred_labels = []
supporting_article_urls = []
explanations = []
for res in results:
claims.append(res.preprocessed_claim)
labels.append(res.original_claim.label)
reasoner_input = ""
for idx, sent in enumerate(res.sentences_for_transformer):
if idx == 10:
break
reasoner_input += " " + sent.preprocessed_text
reasoner_inputs.append(reasoner_input)
pred_labels.append(res.submission_label)
supporting_article_urls.append(", ".join(
res.submission_article_urls.values()))
explanations.append(res.submission_explanation)
results_df = pd.DataFrame(
data={
"claim": claims,
"label": labels,
"reasoner_input": reasoner_inputs,
"predicted": pred_labels,
"article_urls": supporting_article_urls,
"explanation": explanations
})
# Get accuracies
eval_predictions(labels, pred_labels)
# Get datacup score
eval_datacup(raw_claim_dicts, results)
save_results(results_df, "pipeline_test", "full_pipeline")