Пример #1
0
def weibo_subob_rub_neu_classifier(items, batch=RUBBISH_BATCH_COUNT):
    '''
    分类主函数:
    输入数据:weibo(list元素),示例:[[mid,text,...],[mid,text,...]...]
            batch: rubbish filter的参数
    输出数据:label_data(字典元素),示例:{{'mid':类别标签},{'mid':类别标签}...}
            1表示垃圾文本,0表示新闻文本,[2表示中性文本, 已去除],-1表示有极性的文本
    '''
    results = []
    items = rubbish_classifier(items, batch=batch)

    for item in items:
        label = 1
        if item['rub_label'] == 1:
            label = 1 # 垃圾

        else:
            item = subob_classifier(item)
            if item['subob_label'] == 1:
                label = 0 # 客观
            else:
                sentiment = triple_classifier(item)
                if sentiment == 0:
                    # label = 2 # 中性
                    label = cut_mid_weibo(item['content168'])
                else:
                    label = -1 # 有极性

        item['subob_rub_neu_label'] = label
        results.append(item)

    return results
Пример #2
0
def comments_rubbish_clustering_calculation(comments, logger, cluster_num=COMMENT_WORDS_CLUSTER_NUM, \
        cluster_eva_min_size=CLUSTER_EVA_MIN_SIZE, \
        version=COMMENT_CLUSTERING_PROCESS_FOR_CLUTO_VERSION):
    """评论垃圾过滤、聚类
       input: comments
           comment中包含news_id, news_content
       cluster_infos: 聚簇信息
       item_infos:单条信息列表, 数据字段:clusterid、weight、same_from、duplicate
    """
    # 无意义信息的clusterid,包括ad_filter分出来的广告,svm分出的垃圾,主客观分类器分出的新闻
    NON_CLUSTER_ID = 'nonsense'

    # 其他类的clusterid
    OTHER_CLUSTER_ID = 'other'

    # 最小聚类输入信息条数,少于则不聚类
    MIN_CLUSTERING_INPUT = 30

    # 簇信息,主要是簇的特征词信息
    clusters_infos = {'features': dict()}

    # 去除sentiment label clusterid ad_label subob_label rub_label
    clear_keys = ['label', 'ad_label', 'subob_label', 'rub_label', 'weight']
    inputs = []
    for r in comments:
        for key in clear_keys:
            if key in r:
                del r[key]

        inputs.append(r)
    comments = inputs

    # 单条信息list,每条信息存储 clusterid weight sentiment字段
    items_infos = []

    # 数据字段预处理
    inputs = []
    for r in comments:
        r['title'] = ''
        try:
            r['content168'] = r['content168'].encode('utf-8')
        except:
            r['content168'] = r['text'].encode('utf-8')
        r['content'] = r['content168']
        r['text'] = r['content168']
        if 'news_content' in r and r['news_content']:
            r['news_content'] = r['news_content'].encode('utf-8')
        else:
            r['news_content'] = ''

        # 简单规则过滤广告
        item = ad_filter(r)
        if item['ad_label'] == 0:
            inputs.append(item)
        else:
            item['clusterid'] = NON_CLUSTER_ID + '_rub'
            items_infos.append(item)

    # svm去除垃圾
    items = rubbish_classifier(inputs)
    inputs = []
    for item in items:
        if item['rub_label'] == 1:
            item['clusterid'] = NON_CLUSTER_ID + '_rub'
            items_infos.append(item)
        else:
            inputs.append(item)

    # 按新闻对评论归类
    results = comment_news(inputs)

    final_inputs = []
    for news_id, _inputs in results.iteritems():
        # 结合新闻,过滤评论
        _inputs = filter_comment(_inputs)
        inputs = [r for r in _inputs if r['rub_label'] == 0]
        inputs_rubbish = [r for r in _inputs if r['rub_label'] == 1]
        for r in inputs_rubbish:
            r['clusterid'] =  NON_CLUSTER_ID + '_rub'
            items_infos.append(r)

        if len(inputs) >= MIN_CLUSTERING_INPUT:
            tfidf_word, input_dict = tfidf_v2(inputs)
            results = choose_cluster(tfidf_word, inputs, \
                    cluster_num=cluster_num, version=version)

            #评论文本聚类
            cluster_text = text_classify(inputs, results, tfidf_word)

            evaluation_inputs = []

            for k, v in enumerate(cluster_text):
                inputs[k]['label'] = v['label']
                inputs[k]['weight'] = v['weight']
                evaluation_inputs.append(inputs[k])

            #簇评价, 权重及簇标签
            recommend_text = cluster_evaluation(evaluation_inputs, min_size=cluster_eva_min_size)
            for label, items in recommend_text.iteritems():
                if label != OTHER_CLUSTER_ID:
                    clusters_infos['features'][label] = results[label]
                    print '11111',results[label]
                    for item in items:
                        item['clusterid'] = label
                        item['weight'] = item['weight']

                    final_inputs.extend(items)
                else:
                    for item in items:
                        item['clusterid'] = OTHER_CLUSTER_ID
                    items_infos.extend(items)
        else:
            # 如果信息条数小于,则直接展示信息列表
            tfidf_word, input_dict = tfidf_v2(inputs)
            uuid_label = str(uuid.uuid4())
            clusters_infos['features'][uuid_label] = [kw for kw, count in tfidf_word]
            print '22222222',clusters_infos['features'][uuid_label]
            for r in inputs:
                r['clusterid'] = uuid_label
                r['weight'] = global_weight_cal_tfidf(tfidf_word, r)

            final_inputs.extend(inputs)

    # 去重,根据子观点类别去重
    cluster_items = dict()
    for r in final_inputs:
        clusterid = r['clusterid']
        try:
            cluster_items[clusterid].append(r)
        except KeyError:
            cluster_items[clusterid] = [r]

    for clusterid, items in cluster_items.iteritems():
        results = duplicate(items)
        items_infos.extend(results)

    return {'cluster_infos': clusters_infos, 'item_infos': items_infos}
Пример #3
0
def comments_sentiment_rubbish_calculation(comments, logger):
    """输入为一堆comments, 字段包括title、content168
       输出:
           item_infos:单条信息列表, 数据字段:sentiment、same_from、duplicate
    """
    # 无意义信息的clusterid,包括ad_filter分出来的广告,svm分出的垃圾,主客观分类器分出的新闻
    NON_CLUSTER_ID = 'nonsense'

    # 有意义的信息clusterid
    MEAN_CLUSTER_ID = 'sentiment'

    # 单条信息list,每条信息存储 clusterid weight sentiment字段
    items_infos = []

    # 去除sentiment label clusterid ad_label subob_label rub_label
    clear_keys = ['sentiment', 'label', 'clusterid', 'ad_label', 'subob_label', 'rub_label', 'weight']
    inputs = []
    for r in comments:
        for key in clear_keys:
            if key in r:
                del r[key]

        inputs.append(r)
    comments = inputs

    # 数据字段预处理
    inputs = []
    for r in comments:
        r['title'] = ''
        try:
            r['content168'] = r['content168'].encode('utf-8')
        except:
            r['content168'] = r['text'].encode('utf-8')
        r['content'] = r['content168']
        r['text'] = r['content168']

        inputs.append(r)

    # 先分中性及3类分类器
    svm_inputs = []
    for r in inputs:
        sentiment = neutral_classifier(r)

        if sentiment != 0:
            sentiment = triple_classifier(r)
            if sentiment == 0:
                svm_inputs.append(r)
            else:
                r['sentiment'] = sentiment
                items_infos.append(r)
        else:
            svm_inputs.append(r)

    # 情绪调整
    senti_modify_inputs = []
    for r in svm_inputs:
        sentiment = mid_sentiment_classify(r['text'])
        if sentiment == -1:
            sentiment = 0 # 中性

        if sentiment != 0:
            r['sentiment'] = sentiment
            items_infos.append(r)
        else:
            r['sentiment'] = sentiment
            senti_modify_inputs.append(r)

    # 新闻分类
    inputs = []
    for r in senti_modify_inputs:
        r = subob_classifier(r)
        if r['subob_label'] == 1:
            # 主客观文本分类
            r['sentiment'] = NON_CLUSTER_ID + '_news' # 新闻
            items_infos.append(r)
        else:
            inputs.append(r)

    # 去垃圾
    items = rubbish_classifier(inputs)
    for item in items:
        if item['rub_label'] == 1:
            # svm去垃圾
            item['sentiment'] = NON_CLUSTER_ID + '_rub'
        else:
            # 简单规则过滤广告
            item = ad_filter(item)
            if item['ad_label'] == 1:
                item['sentiment'] = NON_CLUSTER_ID + '_rub'

        items_infos.append(item)

    # 去重,在一个情绪类别下将文本去重
    sentiment_dict = dict()
    for item in items_infos:
        if 'sentiment' in item:
            sentiment = item['sentiment']
            try:
                sentiment_dict[sentiment].append(item)
            except KeyError:
                sentiment_dict[sentiment] = [item]

    items_infos = []
    for sentiment, items in sentiment_dict.iteritems():
        items_list = duplicate(items)
        items_infos.extend(items_list)

    return {'item_infos': items_infos}
Пример #4
0
def comments_rubbish_clustering_calculation(comments, logger, cluster_num=COMMENT_WORDS_CLUSTER_NUM, \
        cluster_eva_min_size=CLUSTER_EVA_MIN_SIZE, \
        version=COMMENT_CLUSTERING_PROCESS_FOR_CLUTO_VERSION):
    """评论垃圾过滤、聚类
       input: comments
           comment中包含news_id, news_content
       cluster_infos: 聚簇信息
       item_infos:单条信息列表, 数据字段:clusterid、weight、same_from、duplicate
    """
    # 无意义信息的clusterid,包括ad_filter分出来的广告,svm分出的垃圾,主客观分类器分出的新闻
    NON_CLUSTER_ID = 'nonsense'

    # 其他类的clusterid
    OTHER_CLUSTER_ID = 'other'

    # 最小聚类输入信息条数,少于则不聚类
    MIN_CLUSTERING_INPUT = 30

    # 簇信息,主要是簇的特征词信息
    clusters_infos = {'features': dict()}

    # 去除sentiment label clusterid ad_label subob_label rub_label
    clear_keys = ['label', 'ad_label', 'subob_label', 'rub_label', 'weight']
    inputs = []
    for r in comments:
        for key in clear_keys:
            if key in r:
                del r[key]

        inputs.append(r)
    comments = inputs

    # 单条信息list,每条信息存储 clusterid weight sentiment字段
    items_infos = []

    # 数据字段预处理
    inputs = []
    for r in comments:
        r['title'] = ''
        r['content168'] = r['content168'].encode('utf-8')
        r['content'] = r['content168']
        r['text'] = r['content168']
        if 'news_content' in r and r['news_content']:
            r['news_content'] = r['news_content'].encode('utf-8')
        else:
            r['news_content'] = ''

        # 简单规则过滤广告
        item = ad_filter(r)
        if item['ad_label'] == 0:
            inputs.append(item)
        else:
            item['clusterid'] = NON_CLUSTER_ID + '_rub'
            items_infos.append(item)

    # svm去除垃圾
    items = rubbish_classifier(inputs)
    inputs = []
    for item in items:
        if item['rub_label'] == 1:
            item['clusterid'] = NON_CLUSTER_ID + '_rub'
            items_infos.append(item)
        else:
            inputs.append(item)

    # 按新闻对评论归类
    results = comment_news(inputs)

    final_inputs = []
    for news_id, _inputs in results.iteritems():
        # 结合新闻,过滤评论
        _inputs = filter_comment(_inputs)
        inputs = [r for r in _inputs if r['rub_label'] == 0]
        inputs_rubbish = [r for r in _inputs if r['rub_label'] == 1]
        for r in inputs_rubbish:
            r['clusterid'] =  NON_CLUSTER_ID + '_rub'
            items_infos.append(r)

        if len(inputs) >= MIN_CLUSTERING_INPUT:
            tfidf_word, input_dict = tfidf_v2(inputs)
            results = choose_cluster(tfidf_word, inputs, \
                    cluster_num=cluster_num, version=version)

            #评论文本聚类
            cluster_text = text_classify(inputs, results, tfidf_word)

            evaluation_inputs = []

            for k, v in enumerate(cluster_text):
                inputs[k]['label'] = v['label']
                inputs[k]['weight'] = v['weight']
                evaluation_inputs.append(inputs[k])

            #簇评价, 权重及簇标签
            recommend_text = cluster_evaluation(evaluation_inputs, min_size=cluster_eva_min_size)
            for label, items in recommend_text.iteritems():
                if label != OTHER_CLUSTER_ID:
                    clusters_infos['features'][label] = results[label]

                    for item in items:
                        item['clusterid'] = label
                        item['weight'] = item['weight']

                    final_inputs.extend(items)
                else:
                    for item in items:
                        item['clusterid'] = OTHER_CLUSTER_ID
                    items_infos.extend(items)
        else:
            # 如果信息条数小于,则直接展示信息列表
            tfidf_word, input_dict = tfidf_v2(inputs)
            uuid_label = str(uuid.uuid4())
            clusters_infos['features'][uuid_label] = [kw for kw, count in tfidf_word]
            for r in inputs:
                r['clusterid'] = uuid_label
                r['weight'] = global_weight_cal_tfidf(tfidf_word, r)

            final_inputs.extend(inputs)

    # 去重,根据子观点类别去重
    cluster_items = dict()
    for r in final_inputs:
        clusterid = r['clusterid']
        try:
            cluster_items[clusterid].append(r)
        except KeyError:
            cluster_items[clusterid] = [r]

    for clusterid, items in cluster_items.iteritems():
        results = duplicate(items)
        items_infos.extend(results)

    return {'cluster_infos': clusters_infos, 'item_infos': items_infos}
Пример #5
0
def comments_sentiment_rubbish_calculation(comments, logger):
    """输入为一堆comments, 字段包括title、content168
       输出:
           item_infos:单条信息列表, 数据字段:sentiment、same_from、duplicate
    """
    # 无意义信息的clusterid,包括ad_filter分出来的广告,svm分出的垃圾,主客观分类器分出的新闻
    NON_CLUSTER_ID = 'nonsense'

    # 有意义的信息clusterid
    MEAN_CLUSTER_ID = 'sentiment'

    # 单条信息list,每条信息存储 clusterid weight sentiment字段
    items_infos = []

    # 去除sentiment label clusterid ad_label subob_label rub_label
    clear_keys = ['sentiment', 'label', 'clusterid', 'ad_label', 'subob_label', 'rub_label', 'weight']
    inputs = []
    for r in comments:
        for key in clear_keys:
            if key in r:
                del r[key]

        inputs.append(r)
    comments = inputs

    # 数据字段预处理
    inputs = []
    for r in comments:
        r['title'] = ''
        r['content168'] = r['content168'].encode('utf-8')
        r['content'] = r['content168']
        r['text'] = r['content168']

        inputs.append(r)

    # 先分中性及3类分类器
    svm_inputs = []
    for r in inputs:
        sentiment = neutral_classifier(r)

        if sentiment != 0:
            sentiment = triple_classifier(r)
            if sentiment == 0:
                svm_inputs.append(r)
            else:
                r['sentiment'] = sentiment
                items_infos.append(r)
        else:
            svm_inputs.append(r)

    # 情绪调整
    senti_modify_inputs = []
    for r in svm_inputs:
        sentiment = mid_sentiment_classify(r['text'])
        if sentiment == -1:
            sentiment = 0 # 中性

        if sentiment != 0:
            r['sentiment'] = sentiment
            items_infos.append(r)
        else:
            r['sentiment'] = sentiment
            senti_modify_inputs.append(r)

    # 新闻分类
    inputs = []
    for r in senti_modify_inputs:
        r = subob_classifier(r)
        if r['subob_label'] == 1:
            # 主客观文本分类
            r['sentiment'] = NON_CLUSTER_ID + '_news' # 新闻
            items_infos.append(r)
        else:
            inputs.append(r)

    # 去垃圾
    items = rubbish_classifier(inputs)
    for item in items:
        if item['rub_label'] == 1:
            # svm去垃圾
            item['sentiment'] = NON_CLUSTER_ID + '_rub'
        else:
            # 简单规则过滤广告
            item = ad_filter(item)
            if item['ad_label'] == 1:
                item['sentiment'] = NON_CLUSTER_ID + '_rub'

        items_infos.append(item)

    # 去重,在一个情绪类别下将文本去重
    sentiment_dict = dict()
    for item in items_infos:
        if 'sentiment' in item:
            sentiment = item['sentiment']
            try:
                sentiment_dict[sentiment].append(item)
            except KeyError:
                sentiment_dict[sentiment] = [item]

    items_infos = []
    for sentiment, items in sentiment_dict.iteritems():
        items_list = duplicate(items)
        items_infos.extend(items_list)

    return {'item_infos': items_infos}