def __init__(self):
# Load PolyFuzz model for matching. Default: TF-IDF
self.model = PolyFuzz(config.MODEL_MATCHING)
# Load the filters
self.filters: Dict[str, List[Filter]] = self.__load_filters()
def test_custom_model():
custom_matcher = MyModel()
model = PolyFuzz(custom_matcher).match(from_list, to_list)
matches = model.get_matches()
assert isinstance(matches, pd.DataFrame)
assert matches.Similarity.mean() > 0.0
assert len(matches) == 6
assert list(matches.columns) == ['From', 'To', 'Similarity']
def test_base_model(method):
model = PolyFuzz(method).match(from_list, to_list)
matches = model.get_matches()
assert isinstance(matches, pd.DataFrame)
assert matches.Similarity.mean() > 0.3
assert len(matches) == 6
assert list(matches.columns) == ['From', 'To', 'Similarity']
def similaritypolymain(pTrainData, pTestData, pLevel1, pLevel2, pDesc, pFromDir, pToDir, Nbest):
try:
pTrainData = pTrainData[pTrainData[pDesc].notna()]
pTestData = pTestData[pTestData[pDesc].notna()]
pTestData['Intent'], pTestData['Confidence_Level'] = 'Nan','Nan'
pTrainData, __ = traindata(pTrainData, pDesc, pLevel1, pLevel2, pFromDir, pToDir)
pTrainDataDesc = pd.DataFrame(pTrainData[pDesc])
pTrainDataDescUnq = pTrainDataDesc[pDesc].unique().tolist()
pTestDataDescList = list(pTestData[pDesc].values) #need to convert back to a list
model = PolyFuzz("TF-IDF")
model.match(pTestDataDescList, pTrainDataDescUnq, nbest = int(Nbest))
pMatchesDf = model.get_matches()
IntCol = ["To"]
for i in range(1, int(Nbest)-1):
IntCol.append("BestMatch" + "__" + str(i))
pTestData['Intent' + '__' + str(i)] = 'NaN'
SimCol = ['Similarity']
for k in range(1, int(Nbest) - 1):
SimCol.append("Similarity" + "__" + str(k))
pTestData['Confidence_Level'+ '__' + str(k)] = 'NaN'
for i in range(len(IntCol)):
col = str(IntCol[i])
if col != "To":
for j in range(len(pTestData)):
if pMatchesDf[col][j] != None:
pTestData['Intent' + '__' + str(i-1)][j] = pTrainData[np.where(pTrainData[pDesc] == pMatchesDf[col][j], True , False)]['Intent'].values[0]
else:
for j in range(len(pTestData)):
if pMatchesDf[col][j] != None:
pTestData['Intent'][j] = pTrainData[np.where(pTrainData[pDesc] == pMatchesDf[IntCol[i]][j], True , False)]['Intent'].values[0]
for l in range(len(SimCol)):
col = str(SimCol[l])
if col != "Similarity":
for m in range(len(pTestData)):
if pMatchesDf[col][m] != None:
pTestData['Confidence_Level'+ '__' + int(l-1)][m] = pMatchesDf[SimCol[l]][m]
else:
for m in range(len(pTestData)):
if pMatchesDf[SimCol[l]][m] != None:
pTestData['Confidence_Level'][m] = pMatchesDf[SimCol[l]][m]
except Exception as e:
print('*** ERROR[004]: Error in similarity poly main function: ', sys.exc_info()[0],str(e))
print(traceback.format_exc())
utils.movefile(pFromDir, pToDir)
return(-1)
sys.exit(-1)
return(0, pTestData)
def test_fit_model(method):
model = PolyFuzz(method).fit(from_list, to_list)
matches = model.get_matches()
assert isinstance(matches, pd.DataFrame)
assert matches.Similarity.mean() > 0.3
assert len(matches) == 6
assert list(matches.columns) == ['From', 'To', 'Similarity']
results = model.transform(to_list)
if method == "TF-IDF":
key = "TF-IDF"
elif method == "EditDistance":
key = "EditDistance"
else:
key = list(results.keys())[0]
assert isinstance(results[key], pd.DataFrame)
assert results[key].Similarity.sum() > 0
def test_grouper_same_list():
model = PolyFuzz("TF-IDF").match(from_list, from_list)
model.group(link_min_similarity=0.75, group_all_strings=True)
matches = model.get_matches()
assert isinstance(matches, pd.DataFrame)
assert matches.Similarity.mean() > 0.3
assert len(matches) == 6
assert list(matches.columns) == ['From', 'To', 'Similarity', 'Group']
assert model.get_clusters() == {1: ['apples', 'apple', 'appl']}
assert model.get_cluster_mappings() == {'apples': 1, 'apple': 1, 'appl': 1}
def test_grouper(method):
model = PolyFuzz(method).match(from_list, to_list)
model.group(link_min_similarity=0.75)
matches = model.get_matches()
assert isinstance(matches, pd.DataFrame)
assert matches.Similarity.mean() > 0.3
assert len(matches) == 6
assert list(matches.columns) == ['From', 'To', 'Similarity', 'Group']
assert model.get_clusters() == {1: ['apples', 'apple']}
assert model.get_cluster_mappings() == {'apples': 1, 'apple': 1}
from polyfuzz import PolyFuzz
#import polyfuzz
from_list = ["https://www.tatielou.co.uk/apples/sadasda", "https://www.tatielou.co.uk/oranges/sadasda"]
to_list = ["https://www.tatielou.co.uk/apples/", "https://www.tatielou.co.uk/oranges/", "https://www.tatielou.co.uk/pears/"]
from_list = ["apple", "apples", "appl", "recal", "house", "similarity"]
to_list = ["apple", "apples", "mouse"]
model = PolyFuzz("EditDistance")
model.match(from_list, to_list)
# Auto map by Similarity scores
model.get_matches()
# Keep tweets with more than 3 words
relevant_tweetir = relevant_tweetir[relevant_tweetir[
"tweets.full_text"].str.split().apply(lambda x: len(x) > 3)]
# Add period to the end of sentences
relevant_tweetir["tweets.full_text"] = relevant_tweetir[
"tweets.full_text"].apply(punctuate_sent)
relevant_tweetir["tweets.full_text"] = relevant_tweetir[
"tweets.full_text"].str.lower()
print("\nRemove repeated retweets... >80% fast text similarity")
fasttext_embeddings = WordEmbeddings('en-crawl')
fasttext = Embeddings(fasttext_embeddings,
min_similarity=0,
model_id="FastText")
model = PolyFuzz(fasttext)
start = time.time()
indexes_to_remove = []
for topic in relevant_tweetir["topic"].unique():
topic_tweets = relevant_tweetir.loc[relevant_tweetir["topic"] == topic,
"tweets.full_text"]
for index, tweet in topic_tweets.items():
indexes = topic_tweets.index[topic_tweets.index != index]
for ind in indexes:
model.match(tweet.split(), topic_tweets.loc[ind].split())
mean_sim = round(model.get_matches()["Similarity"].mean(), 2)
if mean_sim > 0.8:
indexes_to_remove.append(ind)
break
relevant_tweetir = relevant_tweetir[~relevant_tweetir.index.
else:
c50.warning(
f"""
👹 **Oh! What the Fuzz!** It seems that the crawl you uploaded was not the one I was looking for!
Currently, I only accept Screaming Frog's internal_all.csv file, yet planning to add more crawlers in the future - namely OnCrawl, DeepCrawl and SiteBulb!
Check-out here [where to find it](https://i.imgur.com/HavO4d6.png)
"""
)
st.stop()
dfIndexable = GSCDf.loc[GSCDf["Indexability"] == "Indexable"]
col_one_list = GSCDf["Address"].tolist()
model = PolyFuzz("EditDistance")
SCHEMES = ("http://", "https://")
if start_execution and (uploaded_file is None):
st.wc50.warning("file 1st!")
st.stop()
else:
model.match(linesList, col_one_list)
Polyfuzz = model.get_matches() # Auto map by Similarity scores
Polyfuzz.columns = ["URL to map", "URL match", "Similarity"]
Polyfuzz.index = Polyfuzz.index + 1
#cmapRed = sns.diverging_palette(10, 133, as_cmap=True)
#cmapRedBlue = sns.color_palette("vlag", as_cmap=True)
def test_multiple_models():
tfidf_matcher = TFIDF(n_gram_range=(3, 3),
min_similarity=0,
model_id="TF-IDF")
tfidf_large_matcher = TFIDF(n_gram_range=(3, 6), min_similarity=0)
base_edit_matcher = EditDistance(n_jobs=1)
ratio_matcher = EditDistance(n_jobs=1, scorer=fuzz.ratio)
rapidfuzz_matcher = RapidFuzz(n_jobs=1)
matchers = [
tfidf_matcher, tfidf_large_matcher, base_edit_matcher, ratio_matcher,
rapidfuzz_matcher
]
model = PolyFuzz(matchers).match(from_list, to_list)
# Test if correct matches are found
for model_id in model.get_ids():
assert model_id in model.get_matches().keys()
assert isinstance(model.get_matches(model_id), pd.DataFrame)
assert len(model.get_matches()) == len(matchers)
# Test if error is raised when accessing clusters before creating them
with pytest.raises(ValueError):
model.get_clusters()
with pytest.raises(ValueError):
model.get_cluster_mappings()
# Test if groupings are found
model.group()
for model_id in model.get_ids():
assert model_id in model.get_cluster_mappings().keys()
assert len(model.get_cluster_mappings()) == len(matchers)
sleep(crawl_delay)
df_final['Status Code'] = my_list
# drop urls if already redirected
df_final = df_final[~df_final["Status Code"].isin(["301"])]
df_final = df_final[~df_final["Status Code"].isin(["302"])]
print("Automatically Mapping URLs ..")
# create lists from dfs
df_final_list = list(df_final["Address"])
df_sf_list = list(df_sf["Address"])
# instantiate PolyFuzz model, choose TF-IDF as the similarity measure and match the two lists.
model = PolyFuzz("TF-IDF").match(df_final_list, df_sf_list)
# make the polyfuzz dataframe
df_matches = model.get_matches()
count_row = df_final.shape[0]
print("Total Opportunity:", count_row, "URLs")
df_stats = pd.merge(df_matches,
df_final,
left_on="From",
right_on="Address",
how="inner")
# sort on similarity
df_stats = df_stats.sort_values(by="Similarity", ascending=False)
def test_wrongbase_model(method):
with pytest.raises(ValueError):
model = PolyFuzz(method).match(from_list, to_list)
def similaritypolymain(pTrainData, pTestData, pAsg, pDesc, pDate, Nbest):
try:
pTrainData = pTrainData[pTrainData[pDesc].notna()]
pTestData = pTestData[pTestData[pDesc].notna()]
pTestData['Assignee_Group_Pred'], pTestData['Confidence_Level'] = 'Nan', float(0.0)
pTrainDataDesc = pd.DataFrame(pTrainData[pDesc])
pFeaList = []
pFeaList = pTrainData['Features'].tolist() + pTestData['Features'].tolist()
pFeaUnqList = list(set(pFeaList))
pMatchData, pData, pTestAppendDf, = [], [], []
pMatchesDf, pTestMatchData, pTestDf = pd.DataFrame(),pd.DataFrame(), pd.DataFrame()
for i in range(len(pFeaUnqList)):
ToData, FromData = pd.DataFrame(), pd.DataFrame()
FromData = pTrainData.loc[pTrainData['Features'] == pFeaUnqList[i]]
ToData = pTestData.loc[pTestData['Features'] == pFeaUnqList[i]]
model = PolyFuzz("TF-IDF")
pTestAppendDf.append(ToData)
if len(ToData[pDesc].tolist()) and len(FromData[pDesc].tolist()) >= 1:
model.match(list(ToData[pDesc].values), FromData[pDesc].unique().tolist(), nbest = int(Nbest))
Matches = model.get_matches()
pMatchData.append(Matches)
pData.append(ToData)
pMatchesDf = pd.concat(pMatchData)
pTestMatchData = pd.concat(pData)
pTestDf = pd.concat(pTestAppendDf)
pMatchesDf.reset_index(inplace=True)
del pMatchesDf['index']
pTestMatchData.reset_index(inplace=True)
del pTestMatchData['index']
pTestDf.reset_index(inplace=True)
del pTestDf['index']
pTestConcatData = pd.concat([pTestMatchData,pMatchesDf], axis = 1)
IntCol = ["To"]
for i in range(1, int(Nbest)-1):
IntCol.append("BestMatch" + "__" + str(i))
pTestMatchData['Assignee_Group_Pred' + '__' + str(i)] = 'NaN'
SimCol = ['Similarity']
for k in range(1, int(Nbest) - 1):
SimCol.append("Similarity" + "__" + str(k))
pTestMatchData['Confidence_Level'+ '__' + str(k)] = 'NaN'
for i in range(len(IntCol)):
col = str(IntCol[i])
if col != "To":
pTestAppendFea = []
for p in range(len(pFeaUnqList)):
pTrainFeaData, pTestFeaData = pd.DataFrame(), pd.DataFrame()
pTrainFeaData = pTrainData.loc[pTrainData['Features'] == pFeaUnqList[p]]
pTestFeaData = pTestConcatData.loc[pTestConcatData['Features'] == pFeaUnqList[p]]
pTestFeaData.reset_index(inplace=True)
del pTestFeaData['index']
if len(pTestFeaData) and len(pTrainFeaData)> 0:
for j in range(len(pTestFeaData)):
if pMatchesDf[col][j] != None:
if len(pTrainFeaData[np.where(pTrainFeaData[pDesc] == pTestFeaData[IntCol[i]][j], True , False)][pAsg].values) != 0:
pTestFeaData['Assignee_Group_Pred' + '__' + str(i-1)][j] = pTrainFeaData[np.where(pTrainFeaData[pDesc] == pTestFeaData[col][j], True , False)][pAsg].values[0]
else:
pTestAppendFea = []
for p in range(len(pFeaUnqList)):
pTrainFeaData, pTestFeaData = pd.DataFrame(), pd.DataFrame()
pTrainFeaData = pTrainData.loc[pTrainData['Features'] == pFeaUnqList[p]]
pTestFeaData = pTestConcatData.loc[pTestConcatData['Features'] == pFeaUnqList[p]]
pTestFeaData.reset_index(inplace=True)
del pTestFeaData['index']
if len(pTestFeaData) and len(pTrainFeaData)> 0:
for j in range(len(pTestFeaData)):
if pTestFeaData[col][j] != None:
if len(pTrainFeaData[np.where(pTrainFeaData[pDesc] == pTestFeaData[IntCol[i]][j], True , False)][pAsg].values) != 0:
pTestFeaData['Assignee_Group_Pred'][j] = pTrainFeaData[np.where(pTrainFeaData[pDesc] == pTestFeaData[IntCol[i]][j], True , False)][pAsg].values[0]
else:
pTestFeaData['Assignee_Group_Pred'][j] = None
pTestAppendFea.append(pTestFeaData)
pTestFeaDf = pd.concat(pTestAppendFea)
pTestFeaDf.reset_index(inplace=True)
del pTestFeaDf['index']
pTestDf.loc[pTestDf['Number'].isin(pTestFeaDf['Number']), ['Confidence_Level', 'Assignee_Group_Pred']] = pTestFeaDf[['Similarity', 'Assignee_Group_Pred']].values
except Exception as e:
print('*** ERROR[004]: Error in similarity poly main function: ', sys.exc_info()[0],str(e))
print(traceback.format_exc())
return(-1)
sys.exit(-1)
return(0, pTestDf)
if RadioMapTo == "To crawled URL":
col_one_list = GSCDf["Address"].tolist()
else:
col_one_list = GSCDf["Title 1"].tolist()
# col_one_listURL = GSCDf["Address"].tolist()
# col_one_list
# st.stop()
if RadioMapAgainst == "all crawled URLs":
GSCDf = dfIndexable
else:
pass
model = PolyFuzz("EditDistance")
gif_path = "mouse.gif"
if start_execution:
########## GIT NOT WORKING (UNBOUND)
# gif_runner = st.image(gif_path)
c1, c2, c3 = st.beta_columns([5, 5, 5])
with c2:
# gif_runner = st.image("mouse.gif")
class Matcher:
def __init__(self):
# Load PolyFuzz model for matching. Default: TF-IDF
self.model = PolyFuzz(config.MODEL_MATCHING)
# Load the filters
self.filters: Dict[str, List[Filter]] = self.__load_filters()
@staticmethod
def __load_filters() -> dict:
"""
Load the filters from filters.toml (by default), create Filter
objects, and return a dictionary of these object classified by
intent.
"""
filters = {}
# Load the raw filter
toml_file = toml.load(config.FILTERS_TOML, _dict=dict)
# Loop over each intent
for intent, raw_filters in toml_file.items():
filter_list = []
# Loop over each filter in this intent
for name, content in raw_filters.items():
# Create and append a Filter object
filter_list.append(
Filter(
name=name,
words=content['words'],
regex=content['regex'],
threshold=content['threshold']
)
)
# Save the filters to the main dictionary
filters[intent] = filter_list
return filters
def get_keywords(self, text: str, intent: str) -> dict:
keywords = {}
if intent in self.filters:
# Split the text into a list of words
entries = text.split(" ")
for filter_ in self.filters[intent]:
# Math similarities between the filter and the given text
self.model.match(entries, filter_.words)
matches: pd.DataFrame = self.model.get_matches()
try:
# Get the word with the maximum similarity
thresholds = matches[matches['Similarity'] >= filter_.threshold]
keyword = thresholds[thresholds['Similarity'] == thresholds['Similarity'].max()].iloc[0, 0]
except Exception:
# If there's no match, set the filter as None
keywords[filter_.name] = None
else:
# Use the keyword to retrieve and save its chained-data
if result := re.search(filter_.regex % keyword, text):
keywords[filter_.name] = result.group(filter_.name)
else:
keywords[filter_.name] = None
def main(n_actors, news_kg, t_actors, tweets_kg, verbose=True):
print("\n1. REMOVING STOPWORDS...")
n_actors, news_kg = remove_stopwords(n_actors, news_kg)
t_actors, tweets_kg = remove_stopwords(t_actors, tweets_kg)
print("\n2. CLUSTER SIMILARITY...")
start = time.time()
embeddings = TransformerWordEmbeddings("bert-base-multilingual-cased")
bert = Embeddings(embeddings, min_similarity=0, model_id="BERT")
model_bert = PolyFuzz(bert)
paired_clusters, non_paired_clusters = cluster_similarity(
model_bert, n_actors, t_actors)
end = time.time()
print(f"Computation time - {round(end - start, 2)} seconds\n")
if verbose:
print("\nSimilar clusters (news vs. tweets):")
print(paired_clusters)
print("\nClusters with no similar pair (only news):")
print(non_paired_clusters)
print("\n3. MAPPING ACTORS TO CLUSTER VALUES...")
news_kg = map_news_in_kg(news_kg, paired_clusters, non_paired_clusters)
tweets_kg = map_tweets_in_kg(tweets_kg, paired_clusters, t_actors)
news_kg["Redge1"] = news_kg["Redge1"].str.lower()
news_kg["Redge2"] = news_kg["Redge2"].str.lower()
news_kg["node1"] = news_kg["node1"].str.lower()
tweets_kg["Redge1"] = tweets_kg["Redge1"].str.lower()
tweets_kg["Redge2"] = tweets_kg["Redge2"].str.lower()
tweets_kg["node1"] = tweets_kg["node1"].str.lower()
if verbose:
print("\nExisting triples...")
print("NEWS:")
# print(news_kg[["Redge1", "node1", "Redge2"]])
print(news_kg)
print("\nTWEETS:")
# print(tweets_kg[["Redge1", "node1", "Redge2"]])
print(tweets_kg)
print("\n4. COMPARE TRIPLES...")
fasttext_embeddings = WordEmbeddings('en-crawl')
fasttext = Embeddings(fasttext_embeddings,
min_similarity=0,
model_id="FastText")
leven_dist = RapidFuzz(n_jobs=1, model_id="leven")
model_names = ["BERT", "FastText", "leven"]
models = [bert, fasttext, leven_dist]
model = PolyFuzz(models)
start = time.time()
paired_KG_evaluation = pd.DataFrame()
non_paired_KG = pd.DataFrame()
for cluster in paired_clusters.itertuples():
news_triples = news_kg[news_kg["edge1"] == cluster.news_key]
tweets_triples = tweets_kg[tweets_kg["edge1"] == cluster.tweets_key]
if tweets_triples.shape[0] == 0:
tweets_triples = tweets_kg[tweets_kg["edge2"] ==
cluster.tweets_key]
if tweets_triples.shape[0] == 0:
non_paired_KG = non_paired_KG.append(tweets_triples,
ignore_index=True)
continue
paired_KG_evaluation = paired_KG_evaluation.append(triples_evaluation(
model, news_triples, tweets_triples, model_names),
ignore_index=True)
if verbose:
print("\nPAIRED CLUSTERS TRIPLES EVALUATION...")
print(paired_KG_evaluation)
end = time.time()
print(f"\nComputation time - {round(end - start, 2)} seconds")
# non paired KG evaluation (triples da notícia para os quais não encontro semelhantes nos tweets)
print("\nNON PAIRED CLUSTERS TRIPLES EVALUATION...")
non_paired_KG = non_paired_KG.append(
news_kg[~news_kg["edge1"].isin(paired_clusters["news_key"])],
ignore_index=True)
non_paired_KG_evaluation = triples_evaluation(model, non_paired_KG,
tweets_kg, model_names)
print(non_paired_KG_evaluation)
print("\n5. FINAL KG EVALUATION...")
final_evaluation = paired_KG_evaluation.append(non_paired_KG_evaluation,
ignore_index=True)
print(final_evaluation)
final_leven = round(final_evaluation['leven'].mean(), 3)
final_fasttext = round(final_evaluation['FastText'].mean(), 3)
final_bert = round(final_evaluation['BERT'].mean(), 3)
final_rouge1 = round(final_evaluation['rouge1'].mean(), 3)
print(
f"\nMEAN LEVEN SIMILARITY BETWEEN NEWS AND TWEETS TRIPLES - {final_leven}"
)
print(
f"\nMEAN FAST TEXT SIMILARITY BETWEEN NEWS AND TWEETS TRIPLES - {final_fasttext}"
)
print(
f"\nMEAN BERT SIMILARITY BETWEEN NEWS AND TWEETS TRIPLES - {final_bert}"
)
print(
f"\nMEAN ROUGE1 F1-SCORE BETWEEN NEWS AND TWEETS TRIPLES - {final_rouge1}"
)
return final_evaluation, {
"leven": final_leven,
"FastText": final_fasttext,
"BERT": final_bert,
"ROUGE1": final_rouge1
}