def test_invalid_input2(self):
     jaro('MARHTA', None)
 def test_invalid_input3(self):
     jaro(None, None)
 def test_valid_input(self):
     # https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance
     self.assertAlmostEqual(jaro('MARTHA', 'MARHTA'), 0.9444444444444445)
     self.assertAlmostEqual(jaro('DWAYNE', 'DUANE'), 0.8222222222222223)
     self.assertAlmostEqual(jaro('DIXON', 'DICKSONX'), 0.7666666666666666)
 def test_invalid_input1(self):
     jaro(None, 'MARHTA')
Ejemplo n.º 5
0
 def test_invalid_input3(self):
     jaro(None, None)
Ejemplo n.º 6
0
def generate_feature(file_name):
    lines = stage3_helper.read_file(file_name)

    features = []
    labels = []

    all_names = []
    for line in lines:
        json1, json2, label = stage3_helper.read_jsons_label_from_line(line)
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_name)
        all_names.append(tokenizers.whitespace(string1))
        all_names.append(tokenizers.whitespace(string2))

    for line in lines:
        json1, json2, label = stage3_helper.read_jsons_label_from_line(line)

        feature = []

        # TODO: Add more features and optimize features.

        # product_type
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_type)
        string1 = string1.lower()
        string2 = string2.lower()
        feature.append(
            simfunctions.jaccard(tokenizers.whitespace(string1),
                                 tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.jaro_winkler(string1, string2, prefix_weight=0.1))
        feature.append(
            simfunctions.jaro(
                tokenizers.whitespace(string1)[0],
                tokenizers.whitespace(string2)[0]))
        # if len(string1) == len(string2):
        #     feature.append(simfunctions.hamming_distance(string1, string2))
        # else:
        #     feature.append(5)
        feature.append(
            simfunctions.cosine(tokenizers.whitespace(string1),
                                tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.overlap_coefficient(tokenizers.whitespace(string1),
                                             tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.monge_elkan(tokenizers.whitespace(string1),
                                     tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.tfidf(tokenizers.whitespace(string1),
                               tokenizers.whitespace(string2)))
        feature.append(len(string1))
        feature.append(len(string2))
        feature.append(len(string1) - len(string2))
        feature.append(len(tokenizers.whitespace(string1)))
        feature.append(len(tokenizers.whitespace(string2)))
        # product_name
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_name)
        string1 = string1.lower()
        string2 = string2.lower()
        feature.append(
            simfunctions.jaccard(tokenizers.whitespace(string1),
                                 tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.jaro_winkler(string1, string2, prefix_weight=0.1))
        if len(string1) == len(string2):
            feature.append(simfunctions.hamming_distance(string1, string2))
        else:
            feature.append(5)
        feature.append(
            simfunctions.cosine(tokenizers.whitespace(string1),
                                tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.overlap_coefficient(tokenizers.whitespace(string1),
                                             tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.monge_elkan(tokenizers.whitespace(string1),
                                     tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.tfidf(tokenizers.whitespace(string1),
                               tokenizers.whitespace(string2)))
        feature.append(len(string1))
        feature.append(len(string2))
        feature.append(len(string1) - len(string2))
        feature.append(
            simfunctions.jaro(
                tokenizers.whitespace(string1)[0],
                tokenizers.whitespace(string2)[0]))
        feature.append(len(tokenizers.whitespace(string1)))
        feature.append(len(tokenizers.whitespace(string2)))

        # product_segment
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_segment)
        string1 = string1.lower()
        string2 = string2.lower()
        feature.append(
            simfunctions.jaccard(tokenizers.qgram(string1, 3),
                                 tokenizers.qgram(string2, 3)))
        feature.append(
            simfunctions.jaro_winkler(string1, string2, prefix_weight=0.1))
        # if len(string1) == len(string2):
        #     feature.append(simfunctions.hamming_distance(string1, string2))
        # else:
        #     feature.append(5)
        feature.append(
            simfunctions.cosine(tokenizers.whitespace(string1),
                                tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.overlap_coefficient(tokenizers.whitespace(string1),
                                             tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.monge_elkan(tokenizers.whitespace(string1),
                                     tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.tfidf(tokenizers.whitespace(string1),
                               tokenizers.whitespace(string2)))
        feature.append(
            simfunctions.jaro(
                tokenizers.whitespace(string1)[0],
                tokenizers.whitespace(string2)[0]))
        feature.append(len(string1))
        feature.append(len(string2))
        feature.append(len(string1) - len(string2))

        feature.append(len(tokenizers.whitespace(string1)))
        feature.append(len(tokenizers.whitespace(string2)))
        # product_long_description
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_long_description)

        if string1 is None or string2 is None:
            feature.append(0.5)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            # feature.append(0)
            # feature.append(0)
            # feature.append(0)
            # feature.append(0)
        else:
            string1 = string1.lower()
            string2 = string2.lower()
            string1 = stage3_helper.cleanhtml(string1)
            string2 = stage3_helper.cleanhtml(string2)
            string1 = stage3_helper.clean_stop_word(string1)
            string2 = stage3_helper.clean_stop_word(string2)
            feature.append(
                simfunctions.jaccard(tokenizers.whitespace(string1),
                                     tokenizers.whitespace(string2)))
            # feature.append(simfunctions.jaro_winkler(string1, string2, prefix_weight=0.1))
            feature.append(
                simfunctions.overlap_coefficient(
                    tokenizers.whitespace(string1),
                    tokenizers.whitespace(string2)))
            # feature.append(simfunctions.monge_elkan(tokenizers.whitespace(string1), tokenizers.whitespace(string2)))
            # feature.append(simfunctions.tfidf(tokenizers.whitespace(string1), tokenizers.whitespace(string2)))
            feature.append(len(string1))
            feature.append(len(string2))
            feature.append(len(string1) - len(string2))

        # product_brand
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_brand)
        string1_name, string2_name = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_name)

        if string1 is None or string1 == '':
            string1 = get_predict_brand(string1_name)

        if string2 is None or string2 == '':
            string2 = get_predict_brand(string2_name)

        if string1 is None or string2 is None:
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
            feature.append(0)
        else:
            feature.append(
                simfunctions.jaccard(tokenizers.whitespace(string1),
                                     tokenizers.whitespace(string2)))
            feature.append(
                simfunctions.jaro_winkler(string1, string2, prefix_weight=0.1))
            feature.append(
                simfunctions.overlap_coefficient(
                    tokenizers.whitespace(string1),
                    tokenizers.whitespace(string2)))
            feature.append(
                simfunctions.monge_elkan(tokenizers.whitespace(string1),
                                         tokenizers.whitespace(string2)))
            feature.append(
                simfunctions.tfidf(tokenizers.whitespace(string1),
                                   tokenizers.whitespace(string2)))
            feature.append(len(string1))
            feature.append(len(string2))
            feature.append(len(string1) - len(string2))
            #feature.append(simfunctions.jaro(tokenizers.whitespace(string1)[0], tokenizers.whitespace(string2)[0]))

        # Contains similar model names.
        string1, string2 = stage3_helper.get_attribute_from_jsons(
            json1, json2, product_name)
        string1 = string1.lower()
        string2 = string2.lower()
        model_strs1 = stage3_helper.find_model_str(string1)
        model_strs2 = stage3_helper.find_model_str(string2)
        # share_model_str = False
        # for model in model_strs1:
        #     if model.lower() in string2.lower():
        #         share_model_str = True
        # for model in model_strs2:
        #     if model.lower() in string1.lower():
        #         share_model_str = True
        # if share_model_str:
        #     feature.append(1)
        # else:
        #     feature.append(0)
        if len(model_strs1) > 0 and len(model_strs2) > 0:
            feature.append(simfunctions.jaccard(model_strs1, model_strs2))
        else:
            feature.append(0.5)
        feature.append(len(model_strs1))
        feature.append(len(model_strs2))
        feature.append(len(model_strs1) - len(string2))

        # Other features.
        common = 0
        common_score = 0.0
        for item in json1:
            if item in json2:
                common += 1
                common_score += simfunctions.jaccard(
                    tokenizers.whitespace(json1[item][0]),
                    tokenizers.whitespace(json2[item][0]))
        common_score = common_score / common
        feature.append(len(json1))
        feature.append(len(json2))
        feature.append(len(json1) - len(json2))
        feature.append(common)
        feature.append(common_score)
        feature.append(len(json.dumps(json1)))
        feature.append(len(json.dumps(json2)))
        feature.append(len(json.dumps(json1)) - len(json.dumps(json2)))
        feature.append(
            simfunctions.jaccard(tokenizers.whitespace(json.dumps(json1)),
                                 tokenizers.whitespace(json.dumps(json2))))

        # Add one feature and label.
        features.append(feature)
        labels.append(stage3_helper.get_01_from_label(label))

    return features, labels, lines
Ejemplo n.º 7
0
 def test_invalid_input1(self):
     jaro(None, 'MARHTA')
Ejemplo n.º 8
0
 def test_invalid_input2(self):
     jaro('MARHTA', None)
Ejemplo n.º 9
0
 def time_medium_long(self):
     simfunctions.jaro(_medium_string_1, _long_string_1)
Ejemplo n.º 10
0
 def test_valid_input(self):
     # https://en.wikipedia.org/wiki/Jaro%E2%80%93Winkler_distance
     self.assertAlmostEqual(jaro('MARTHA', 'MARHTA'), 0.9444444444444445)
     self.assertAlmostEqual(jaro('DWAYNE', 'DUANE'), 0.8222222222222223)
     self.assertAlmostEqual(jaro('DIXON', 'DICKSONX'), 0.7666666666666666)
Ejemplo n.º 11
0
 def time_short_long(self):
     simfunctions.jaro(_short_string_1, _long_string_1)
Ejemplo n.º 12
0
 def time_short_medium(self):
     simfunctions.jaro(_short_string_1, _medium_string_1)
Ejemplo n.º 13
0
 def time_long_long(self):
     simfunctions.jaro(_long_string_1, _long_string_2)
Ejemplo n.º 14
0
 def time_medium_medium(self):
     simfunctions.jaro(_medium_string_1, _medium_string_2)
Ejemplo n.º 15
0
 def time_short_short(self):
     simfunctions.jaro(_short_string_1, _short_string_2)