def test_text_similarity_jaccard_on_matrix():
matrix_1 = np.array([["hello", "columbia university"],
["bye", "nyu"]])
matrix_2 = np.array([["helli", "columbia"],
["bye", "new york"]])
tmp = similarities().text_similarity_on_matrix(matrix_1, matrix_2, method="jaccard")
assert (tmp.shape==(4,2))
def test_vector_similarity_on_matrix():
matrix_1 = np.array([[[1,1,1], [1,2,3],[1,3,1]],
[[3,6,7],[2,3,1],[1,1,1]]])
matrix_2 = np.array([[[2,2,2],[2,3,4],[1,1,1]]])
tmp = similarities().vector_similarity_on_matrix(matrix_1,matrix_2)
desired = np.array([[1, 0.993,0.87],[0.953, 0.844, 1]])
assert(np.array_equal(tmp,desired))
def test_text_similarity_jaro_on_matrix():
matrix_1 = np.array([["hello", "columbia university"],
["bye", "nyu"]])
matrix_2 = np.array([["helli", "columbia"],
["bye", "new york"]])
tmp = similarities().text_similarity_on_matrix(matrix_1, matrix_2, method="jaro_winkler")
desired = np.array([[0.92, 0.88421053],
[0.51111111, 0.3998538],
[0.51111111, 0.48611111],
[1., 0.63888889]])
assert (np.array_equal(tmp, desired))
def test_text_similarity_on_matrix():
matrix_1 = np.array([["hello","columbia university"],
["bye","nyu"]])
matrix_2 = np.array([["helli","columbia"],
["bye","new york"]])
tmp = similarities().text_similarity_on_matrix(matrix_1,matrix_2)
desired = np.array([[ 1., 11.],
[ 5., 17.],
[ 5., 7.],
[ 0., 6.]])
assert (np.array_equal(tmp, desired))
# save for later use to generate labels
df1_id_col = df1[df1_id]
df2_id_col = df2[df2_id]
# drop id columns because we don't need to compute id similarity
df1 = df1.drop(columns=[df1_id])
df2 = df2.drop(columns=[df2_id])
processed_data = Preprocessing().overall_preprocess(df1.drop(columns=['description']), df2.drop(columns=['description']),
special_columns=['title','manufacturer'],
word_embedding_model='none') # may take a while bc loading pretrained word embedding model
num_matrix_1, num_matrix_2 = processed_data["numerical"][0],processed_data["numerical"][1]
spc_matrix_1, spc_matrix_2 = processed_data["special_fields"][0],processed_data["special_fields"][1]
num_final_data = similarities().numerical_similarity_on_matrix(num_matrix_1,num_matrix_2)
spc_final_data = similarities().text_similarity_on_matrix(spc_matrix_1,spc_matrix_2,method='jaccard')
df1['key'] = 0
df2['key'] = 0
merged = pd.merge(df1, df2, on='key')[['description_x', 'description_y']]
'''
train-test split
'''
non_empty = []
for m in num_final_data, spc_final_data:#, embed_mean_data, embed_max_data, embed_min_data:
if m.size !=0:
non_empty.append(m)
# Drop id columns because we don't need to compute id similarity
df1 = df1.drop(columns=[df1_id])
df2 = df2.drop(columns=[df2_id])
processed_data = Preprocessing().overall_preprocess(
df1.drop(columns=['description']),
df2.drop(columns=['description']),
special_columns=['title', 'manufacturer'],
word_embedding_model='none'
) # may take a while bc loading pretrained word embedding model
num_matrix_1, num_matrix_2 = processed_data["numerical"][0], processed_data[
"numerical"][1]
spc_matrix_1, spc_matrix_2 = processed_data["special_fields"][
0], processed_data["special_fields"][1]
num_final_data = similarities().numerical_similarity_on_matrix(
num_matrix_1, num_matrix_2)
spc_final_data_0 = similarities().text_similarity_on_matrix(spc_matrix_1,
spc_matrix_2,
method='jaccard')
spc_final_data_1 = similarities().text_similarity_on_matrix(
spc_matrix_1, spc_matrix_2, method='lavenshtein')
spc_final_data_2 = similarities().text_similarity_on_matrix(
spc_matrix_1, spc_matrix_2, method='jaro_winkler')
df1['key'] = 0
df2['key'] = 0
merged = pd.merge(df1, df2, on='key')[['description_x', 'description_y']]
'''
train-test split
'''
non_empty = []
embedding_weight='tfidf')
# may take a while bc loading pretrained word embedding model
'''
get numerical data
'''
# need fix addressZip and not to see it as numeric
num_matrix_1, num_matrix_2 = processed_data["numerical"][0], processed_data[
"numerical"][1]
embed_matrix_1, embed_matrix_2 = processed_data["word_embedding_fields"][
0], processed_data["word_embedding_fields"][1]
spc_matrix_1, spc_matrix_2 = processed_data["special_fields"][
0], processed_data["special_fields"][1]
'''
calculate similarities
'''
num_final_data = similarities().numerical_similarity_on_matrix(
num_matrix_1, num_matrix_2)
embed_tfidf_data = similarities().vector_similarity_on_matrix(
embed_matrix_1, embed_matrix_2)
#embed_mean_data = similarities().vector_similarity_on_matrix(embed_matrix_1,embed_matrix_2)
#embed_min_data = similarities().vector_similarity_on_matrix(embed_matrix_1,embed_matrix_2)
#embed_max_data = similarities().vector_similarity_on_matrix(embed_matrix_1,embed_matrix_2)
spc_final_data = similarities().text_similarity_on_matrix(
spc_matrix_1, spc_matrix_2)
'''
concatenate all data
'''
# only concatenate non-empty similarity matrices
non_empty = []
for m in num_final_data, spc_final_data, embed_tfidf_data: #, embed_min_data, embed_max_data:#, embed_mean_data,embed_tfidf_data#:
if m.size != 0:
def test_numerical_similarity_on_matrix():
matrix_1 = np.array([[1,2,3,4],[5,6,7,8]])
matrix_2 = np.array([[1,2,3,4]])
tmp = similarities().numerical_similarity_on_matrix(matrix_1,matrix_2,method = "min_max")
assert(np.array_equal(tmp,np.array([[1,1,1,1],[0.2 ,0.333, 0.429, 0.5 ]])))
def test_vector_similarity_on_matrix_empty(): #check case where one of the input matrix is empty
matrix_2 = np.array([[[2,2,2],[2,3,4],[1,1,1]]])
tmp = similarities().vector_similarity_on_matrix(np.array([]),matrix_2)
desired = np.array([[1, 0.993,0.87],[0.953, 0.844, 1]])
assert(tmp.size==0)