Ejemplo n.º 1
0

# ---------------- select_features_by_positive_degree() ----------------
def select_features_by_positive_degree(tsm_positive, tsm_unlabeled, (threshold_pd_word, threshold_speciality, threshold_popularity)):

    total_terms = tsm_positive.get_total_terms()
    total_samples = tsm_positive.get_total_samples()

    selected_terms = {}

    rowidx = 0
    logging.debug(Logger.debug("Calculate PDword. %d samples, %d terms in tsm_positive" % (total_samples, total_terms)))
    for term_id in tsm_positive.term_matrix():
        term_info = tsm_positive.get_term_row(term_id)

        (pd_word, speciality, popularity) = calculate_term_positive_degree(term_id, tsm_positive, tsm_unlabeled, None)
        if pd_word >= threshold_pd_word and speciality >= threshold_speciality and popularity >= threshold_popularity:
            selected_terms[term_id] = (pd_word, speciality, popularity)

        if rowidx % 1000 == 0:
            logging.debug(Logger.debug("feature_selection() %d/%d - pd_word:%.6f speciality:%.6f popularity:%.6f" % (rowidx, total_terms, speciality + popularity, speciality, popularity)))
        rowidx += 1

    return selected_terms


# ---------------- get_terms_positive_degree_by_category() ----------------
def get_terms_positive_degree_by_category(tsm, positive_samples_list, unlabeled_samples_list):
    tsm_positive = tsm.clone(positive_samples_list)
    tsm_unlabeled = tsm.clone(unlabeled_samples_list)
Ejemplo n.º 2
0
    def query_by_id(self, samples_positive, samples_unlabeled, sample_id):
        tsm_positive = samples_positive.tsm
        tsm_unlabeled = samples_unlabeled.tsm

        sensitive_words = {
                ##u"立案":3.0,
                ##u"获刑":3.0,
                ##u"受贿":3.0,
                ##u"有期徒刑":3.0,
                ##u"宣判":3.0,
                ##u"审计":2.0,
                ##u"调查":2.0
                }

        sensitive_terms = self.transform_sensitive_terms(sensitive_words, self.vocabulary)

        try:
            sample_content = samples_unlabeled.db_content.Get(str(sample_id))
            #(_, category, date, title, key, url, content) = msgpack.loads(sample_content)

            (_, category, date, title, key, url, msgext) = decode_sample_meta(sample_content)
            (version, content, (cat1, cat2, cat3)) = msgext

            print "sample id: %d" % (sample_id)
            print "category: %d" % (category)
            print "key: %s" % (key)
            print "url: %s" % (url)
            print "date: %s" % (date)
            print "title: %s" % (title)
            print "---------------- content ----------------"
            #print "%s" % (content)

            sample_terms, term_map = self.vocabulary.seg_content(content)
            print "sample_terms: %d terms_count: %d" % (sample_terms, len(term_map))
            #for term_id in term_map:
            terms_list = sorted_dict_by_values(term_map, reverse=True)
            for (term_id, term_used_in_sample) in terms_list:
                term_text = self.vocabulary.get_term_text(term_id)
                #term_used_in_sample = term_map[term_id]
                print "%s(%d): %d" % (term_text, term_id, term_used_in_sample)


        except KeyError:
            print "Sample %d not found in db_content." % (sample_id)

        db_sm = samples_unlabeled.tsm.open_db_sm()
        try:
            str_sample_info = db_sm.Get(str(sample_id))
            (category, sample_terms, term_map) = msgpack.loads(str_sample_info)
            print ""
            print "---------------- keywords ----------------"
            print ""
            terms = {}
            for term_id in term_map:
                term_text = self.vocabulary.get_term_text(term_id)
                term_used = term_map[term_id]
                (pd_word, speciality, popularity) = calculate_term_positive_degree(term_id, tsm_positive, tsm_unlabeled, sensitive_terms)
                terms[term_id] = (pd_word, speciality, popularity, term_used, term_text)

            terms_list = sorted_dict_by_values(terms, reverse = True)
            for (term_id, (pd_word, speciality, popularity, term_used, term_text)) in terms_list:
                print "%s\t%d\t[%.6f,%.6f,%.6f]\t(id:%d)" % (term_text, term_used, pd_word, speciality, popularity, term_id)

        except KeyError:
            print "Sample %d not found in db_sm." % (sample_id)

        samples_unlabeled.tsm.close_db(db_sm)