Ejemplo n.º 1
0
def get_user_sensitive_words(uid):
    user_sensitive_words_dict = {}
    if RUN_TYPE:
        now_ts = time.time()
        now_date = ts2datetime(now_ts) # 2015-09-22
    else:
        now_date = "2013-09-08"
    ts = datetime2ts(now_date)

    #test
    #ts = datetime2ts('2013-09-08')
    for i in range(1,8):
        ts = ts - 3600*24
        date = ts2datetime(ts).replace('-','')
        results = r_cluster.hget('sensitive_'+str(ts), uid)
        if results:
            sensitive_words_dict = json.loads(results)
            for word in sensitive_words_dict:
                if user_sensitive_words_dict.has_key(word):
                    user_sensitive_words_dict[word] += sensitive_words_dict[word]
                else:
                    user_sensitive_words_dict[word] = sensitive_words_dict[word]
    sort_sensitive_words_dict = sorted(user_sensitive_words_dict.items(), key=lambda x:x[1], reverse=True)

    return sort_sensitive_words_dict
def get_group_user_track(uid):
    results = []
    # step1:get user_portrait activity_geo_dict
    try:
        portrait_result = es_user_portrait.get(
            index=portrait_index_name, doc_type=portrait_index_type, id=uid, _source=False, fields=["activity_geo_dict"]
        )
    except:
        portrait_result = {}
    if portrait_result == {}:
        return "uid is not in user_portrait"
    activity_geo_dict = json.loads(portrait_result["fields"]["activity_geo_dict"][0])
    now_date_ts = datetime2ts(ts2datetime(int(time.time())))
    start_ts = now_date_ts - DAY * len(activity_geo_dict)
    # step2: iter date to get month track
    for geo_item in activity_geo_dict:
        iter_date = ts2datetime(start_ts)
        sort_day_dict = sorted(geo_item.items(), key=lambda x: x[1], reverse=True)
        if sort_day_dict:
            results.append([iter_date, sort_day_dict[0][0]])
        else:
            results.append([iter_date, ""])
        start_ts = start_ts + DAY

    return results
def search_mention(uid, sensitive):
    date = ts2datetime(time.time()).replace('-','')
    stat_results = dict()
    results = dict()
    test_ts = time.time()
    test_ts = datetime2ts('2013-09-07')
    for i in range(0,7):
        ts = test_ts -i*24*3600
        date = ts2datetime(ts).replace('-', '')
        if not sensitive:
            at_temp = r_cluster.hget('at_' + str(date), str(uid))
        else:
            at_temp = r_cluster.hget('sensitive_at_' + str(date), str(uid))
        if not at_temp:
            continue
        else:
            result_dict = json.loads(at_temp)
        for at_uid in result_dict:
            if stat_results.has_key(at_uid):
                stat_results[uid] += result_dict[at_uid]
            else:
                stat_results[uid] = result_dict[at_uid]
    if not stat_results:
        return [None, 0]

    in_status = identify_uid_list_in(result_dict.keys())
    for at_uid in result_dict:
        if at_uid in in_status:
            results[at_uid] = [result_dict[at_uid], '1']
        else:
            results[at_uid] = [result_dict[at_uid], '0']

    sorted_results = sorted(results.items(), key=lambda x:x[1][0], reverse=True)
    return [sorted_results[0:20], len(results)]
Ejemplo n.º 4
0
def get_user_sensitive_words(uid):
    user_sensitive_words_dict = {}
    if RUN_TYPE:
        now_ts = time.time()
        now_date = ts2datetime(now_ts)  # 2015-09-22
    else:
        now_date = "2013-09-08"
    ts = datetime2ts(now_date)

    #test
    #ts = datetime2ts('2013-09-08')
    for i in range(1, 8):
        ts = ts - 3600 * 24
        date = ts2datetime(ts).replace('-', '')
        results = r_cluster.hget('sensitive_' + str(ts), uid)
        if results:
            sensitive_words_dict = json.loads(results)
            for word in sensitive_words_dict:
                if user_sensitive_words_dict.has_key(word):
                    user_sensitive_words_dict[word] += sensitive_words_dict[
                        word]
                else:
                    user_sensitive_words_dict[word] = sensitive_words_dict[
                        word]
    sort_sensitive_words_dict = sorted(user_sensitive_words_dict.items(),
                                       key=lambda x: x[1],
                                       reverse=True)

    return sort_sensitive_words_dict
Ejemplo n.º 5
0
def get_group_user_track(uid):
    results = []
    #step1:get user_portrait activity_geo_dict
    try:
        portrait_result = es_user_portrait.get(index=portrait_index_name, doc_type=portrait_index_type,\
                id=uid, _source=False, fields=['activity_geo_dict'])
    except:
        portrait_result = {}
    if portrait_result == {}:
        return 'uid is not in user_portrait'
    activity_geo_dict = json.loads(
        portrait_result['fields']['activity_geo_dict'][0])
    now_date_ts = datetime2ts(ts2datetime(int(time.time())))
    start_ts = now_date_ts - DAY * len(activity_geo_dict)
    #step2: iter date to get month track
    for geo_item in activity_geo_dict:
        iter_date = ts2datetime(start_ts)
        sort_day_dict = sorted(geo_item.items(),
                               key=lambda x: x[1],
                               reverse=True)
        if sort_day_dict:
            results.append([iter_date, sort_day_dict[0][0]])
        else:
            results.append([iter_date, ''])
        start_ts = start_ts + DAY

    return results
def count_hot_uid(uid, start_time, stop_time):
    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                            {"range":{
                                "timestamp":{
                                    "gte":start_time,
                                    "lt": stop_time
                                }
                            }},
                            {"term": {"root_uid": uid}}
                        ]
                    }
                }
#                "query":{
#                    "bool":{
#                        "should":[
#                        ]
#                    }
#                }
            }
        }
    }


    count = 0
    datetime = ts2datetime(float(stop_time))
    index_name = flow_text_index_name_pre + datetime
    exist_es = es_text.indices.exists(index_name)
    if exist_es:
        count = es_text.count(index=index_name, doc_type=flow_text_index_type, body=query_body)["count"]
    else:
        count = 0

    datetime_1 = ts2datetime(float(start_time))
    if datetime_1 == datetime:
        pass
    else:
        ts = float(stop_time)
        while 1:
            ts = ts-day_time
            datetime = ts2datetime(ts)
            index_name = flow_text_index_name_pre + datetime
            exist_es = es_text.indices.exists(index_name)
            if exist_es:
                count = es_text.count(index=index_name, doc_type=flow_text_index_type, body=query_body)["count"]
            else:
                count += 0
            if datetime_1 == datetime:
                break

    return count
Ejemplo n.º 7
0
def get_user_geo(uid):
    results = []
    user_geo_result = {}
    user_ip_dict = {}
    user_ip_result = {}  # ordinary ip
    user_sensitive_ip_result = {}  # sensitive ip
    now_ts = time.time()
    now_date = ts2datetime(now_ts)  # 2015-09-22
    ts = datetime2ts(now_date)

    #test
    ts = datetime2ts('2013-09-08')
    for i in range(1, 8):
        ts = ts - 3600 * 24
        date = ts2datetime(ts).replace('-', '')
        results = r_cluster.hget('ip_' + str(date), uid)
        sensitive_results = r_cluster.hget('sensitive_ip' + str(date), uid)
        if results:
            ip_results = json.loads(results)
            for ip in ip_results:
                if user_ip_result.has_key(ip):
                    user_ip_result[ip] += ip_results[ip]
                else:
                    user_ip_result[ip] = ip_results[ip]

        if sensitive_results:
            sensitive_ip_results = json.loads(sensitive_results)
            for ip in sensitive_ip_results:
                if user_sensitive_ip_result.has_key(ip):
                    user_sensitive_ip_result[ip] += sensitive_ip_results[ip]
                else:
                    user_sensitive_ip_result[ip] = sensitive_ip_results[ip]

    ordinary_key_set = set(user_ip_result.keys())
    sensitive_key_set = set(user_sensitive_ip_result.keys())
    for key in sensitive_key_set:
        if key in ordinary_key_set:
            user_ip_result[key] += user_sensitive_ip_result[key]
        else:
            user_ip_result[key] = user_sensitive_ip_result[key]

    user_geo_dict = ip2geo(user_ip_result)
    sorted_user_geo_dict = sorted(user_geo_dict.items(),
                                  key=lambda x: x[1],
                                  reverse=True)
    sensitive_user_geo_dict = ip2geo(user_sensitive_ip_result)
    sorted_sensitive_user_geo_dict = sorted(sensitive_user_geo_dict.items(),
                                            key=lambda x: x[1],
                                            reverse=True)

    return_list = []
    return_list = [sorted_user_geo_dict,
                   sorted_sensitive_user_geo_dict]  # total and sensitive
    return return_list
def query_hot_mid(ts, keywords_list, text_type,size=100):
    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                            {"range":{
                                "timestamp":{
                                    "gte":ts - time_interval,
                                    "lt": ts
                                }
                            }},
                            {"terms": {"keywords_string": keywords_list}},
                            {"term": {"message_type": "0"}}
                        ]
                    }
                }
            }
        },
        "aggs":{
            "all_interests":{
                "terms":{"field": "root_mid", "size": size}
            }
        }
    }

    datetime = ts2datetime(ts)
    datetime_1 = ts2datetime(ts-time_interval)
    index_name = flow_text_index_name_pre + datetime
    exist_es = es_text.indices.exists(index_name)
    index_name_1 = flow_text_index_name_pre + datetime_1
    exist_bool_1 = es_text.indices.exists(index_name_1)
    print datetime, datetime_1
    if datetime == datetime_1 and exist_es:
        search_results = es_text.search(index=index_name, doc_type=flow_text_index_type, body=query_body)["aggregations"]["all_interests"]["buckets"]
    elif datetime != datetime_1 and exist_bool_1:
        search_results = es_text.search(index=index_name_1, doc_type=flow_text_index_type, body=query_body)["aggregations"]["all_interests"]["buckets"]
    else:
        search_results = []

    hot_mid_list = []
    if search_results:
        for item in search_results:
            print item
            temp = []
            temp.append(item['key'])
            temp.append(item['doc_count'])
            hot_mid_list.append(temp)

    #print hot_mid_list

    return hot_mid_list
Ejemplo n.º 9
0
def get_user_geo(uid):
    results = []
    user_geo_result = {}
    user_ip_dict = {}
    user_ip_result = {} # ordinary ip
    user_sensitive_ip_result = {} # sensitive ip
    now_ts = time.time()
    now_date = ts2datetime(now_ts) # 2015-09-22
    ts = datetime2ts(now_date)

    #test
    ts = datetime2ts('2013-09-08')
    for i in range(1,8):
        ts = ts - 3600*24
        date = ts2datetime(ts).replace('-','')
        results = r_cluster.hget('ip_'+str(date), uid)
        sensitive_results = r_cluster.hget('sensitive_ip'+str(date), uid)
        if results:
            ip_results = json.loads(results)
            for ip in ip_results:
                if user_ip_result.has_key(ip):
                    user_ip_result[ip] += ip_results[ip]
                else:
                    user_ip_result[ip] = ip_results[ip]

        if sensitive_results:
            sensitive_ip_results = json.loads(sensitive_results)
            for ip in sensitive_ip_results:
                if user_sensitive_ip_result.has_key(ip):
                    user_sensitive_ip_result[ip] += sensitive_ip_results[ip]
                else:
                    user_sensitive_ip_result[ip] = sensitive_ip_results[ip]

    ordinary_key_set = set(user_ip_result.keys())
    sensitive_key_set = set(user_sensitive_ip_result.keys())
    for key in sensitive_key_set:
        if key in ordinary_key_set:
            user_ip_result[key] += user_sensitive_ip_result[key]
        else:
            user_ip_result[key] = user_sensitive_ip_result[key]

    user_geo_dict = ip2geo(user_ip_result)
    sorted_user_geo_dict = sorted(user_geo_dict.items(), key=lambda x:x[1], reverse=True)
    sensitive_user_geo_dict = ip2geo(user_sensitive_ip_result)
    sorted_sensitive_user_geo_dict = sorted(sensitive_user_geo_dict.items(), key=lambda x:x[1], reverse=True)


    return_list = []
    return_list = [sorted_user_geo_dict, sorted_sensitive_user_geo_dict] # total and sensitive
    return return_list
def get_influence_content(uid, timestamp_from, timestamp_to):
    weibo_list = []
    # split timestamp range to new_range_dict_list
    from_date_ts = datetime2ts(ts2datetime(timestamp_from))
    to_date_ts = datetime2ts(ts2datetime(timestamp_to))
    new_range_dict_list = []
    if from_date_ts != to_date_ts:
        iter_date_ts = from_date_ts
        while iter_date_ts < to_date_ts:
            iter_next_date_ts = iter_date_ts + DAY
            new_range_dict_list.append({"range": {"timestamp": {"gte": iter_date_ts, "lt": iter_next_date_ts}}})
            iter_date_ts = iter_next_date_ts
        if new_range_dict_list[0]["range"]["timestamp"]["gte"] < timestamp_from:
            new_range_dict_list[0]["range"]["timestamp"]["gte"] = timestamp_from
        if new_range_dict_list[-1]["range"]["timestamp"]["lt"] > timestamp_to:
            new_range_dict_list[-1]["range"]["timestamp"]["lt"] = timestamp_to
    else:
        new_range_dict_list = [{"range": {"timestamp": {"gte": timestamp_from, "lt": timestamp_to}}}]
    # iter date to search flow_text
    iter_result = []
    for range_item in new_range_dict_list:
        range_from_ts = range_item["range"]["timestamp"]["gte"]
        range_from_date = ts2datetime(range_from_ts)
        flow_text_index_name = flow_text_index_name_pre + range_from_date
        query = []
        query.append({"term": {"uid": uid}})
        query.append(range_item)
        try:
            flow_text_exist = es_flow_text.search(
                index=flow_text_index_name,
                doc_type=flow_text_index_type,
                body={"query": {"bool": {"must": query}}, "sort": [{"timestamp": "asc"}]},
            )["hits"]["hits"]
        except:
            flow_text_exist = []
        iter_result.extend(flow_text_exist)
    # get weibo list
    for item in flow_text_exist:
        source = item["_source"]
        weibo = {}
        weibo["timestamp"] = ts2date(source["timestamp"])
        weibo["ip"] = source["ip"]
        weibo["text"] = source["text"]
        if source["geo"]:
            weibo["geo"] = "\t".join(source["geo"].split("&"))
        else:
            weibo["geo"] = ""
        weibo_list.append(weibo)

    return weibo_list
def get_network(task_exist):
    task_name = task_exist['task_name']
    submit_date = task_exist['submit_date']
    submit_ts = date2ts(submit_date)

    time_segment = 24*3600
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    now_date_ts = datetime2ts(now_date)
    #test
    now_date_ts = datetime2ts('2013-09-07')
    iter_date_ts = now_date_ts
    iter_count = 1
    date_list = []
    top_list_dict = {}
    while True:
        if iter_count >= 8 or iter_date_ts < submit_ts:
            break
        iter_date = ts2datetime(iter_date_ts)
        date_list.append(iter_date)
        key = 'inner_' + str(iter_date)
        try:
            task_date_result = es.get(index=monitor_index_name, doc_type=task_name, id=key)['_source']
        except:
            task_date_result = {}
        #print 'task_name, key, task_date_result:', task_name, key, task_date_result
        iter_field = ['top1', 'top2', 'top3', 'top4', 'top5']
        for field in iter_field:
            user_count_item = json.loads(task_date_result[field])
            uid = user_count_item[0]
            uname = uid2uname(uid)
            count = user_count_item[1]
            try:
                top_list_dict[field].append([uid, uname, count])
            except:
                top_list_dict[field] = [[uid, uname, count]]
        
        iter_date_ts -= time_segment
        # get inner-retweet group from es---field: inner_graph
        '''
        try:
            inner_graph = json.loads(task_date_result['inner_graph'])
        except:
            inner_graph = {}
        '''

    abnormal_index = compute_inner_polarization(top_list_dict)
    
    return [date_list, top_list_dict, abnormal_index]
Ejemplo n.º 12
0
def ajax_upload_track_file():
    results = {}
    upload_data = request.form['upload_data']
    task_name = request.form['task_name']
    state = request.args.form['state']
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    now_date_ts = datetime2ts(now_date)
    time_segment = int((now_ts - now_Date_ts) / 900) + 1
    trans_ts = now_date_ts + time_segment * 900
    line_list = upload_data.split('\n')
    input_data = {}
    #submit task and start time is 15min multiple
    input_data['submit_date'] = trans_ts
    input_data['task_name'] = task_name
    uid_list = []
    for line in line_list:
        uid = line[:10]
        if len(uid) == 10:
            uid_list.append(uid)
    input_data['uid_list'] = uid_list
    input_data[
        'status'] = 1  # status show the track task is doing or end; doing 1, end 0
    input_data['count'] = len(uid_list)
    status = submit_track_task(input_data)
    return json.dumps(status)
Ejemplo n.º 13
0
def ajax_upload_track_file():
    results = {}
    upload_data = request.form['upload_data']
    task_name = request.form['task_name']
    state = request.args.form['state']
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    now_date_ts = datetime2ts(now_date)
    time_segment = int((now_ts - now_Date_ts) / 900) + 1
    trans_ts = now_date_ts + time_segment * 900
    line_list = upload_data.split('\n')
    input_data = {}
    #submit task and start time is 15min multiple
    input_data['submit_date'] = trans_ts
    input_data['task_name'] = task_name
    uid_list = []
    for line in line_list:
        uid = line[:10]
        if len(uid)==10:
            uid_list.append(uid)
    input_data['uid_list'] = uid_list
    input_data['status'] = 1 # status show the track task is doing or end; doing 1, end 0
    input_data['count'] = len(uid_list)
    status = submit_track_task(input_data)
    return json.dumps(status)
Ejemplo n.º 14
0
def influence_distribute():

    row = [0, 200, 500, 700, 900, 1100, 10000]
    result = []
    ts = time.time()
    ts = datetime2ts('2013-09-08') # test
    ts = ts - 8*3600*24
    for j in range(7):
        detail = []
        ts += 3600*24
        date = ts2datetime(ts).replace('-', '')
        for i in range(6):
            low_limit = row[i]
            upper_limit = row[i+1]
            query_body = {
                "query": {
                    "filtered": {
                        "filter": {
                            "range": {
                                date: {
                                    "gte": low_limit,
                                    "lt": upper_limit
                                }
                            }
                        }
                    }
                }
            }
            number = es.count(index='copy_sensitive_user_portrait', doc_type="user", body=query_body)['count']
            detail.append(number)
        result.append(detail)
    return [row, result]
Ejemplo n.º 15
0
def show_detect_task(submit_user):
    results = []
    query = [{
        'match': {
            'task_type': 'detect'
        }
    }, {
        'term': {
            'submit_user': submit_user
        }
    }]
    try:
        search_results = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
                body={'query':{'bool':{'must':query}}, 'sort':[{'submit_date': 'desc'}], 'size':MAX_VALUE})['hits']['hits']
    except:
        search_results = []
    for group_item in search_results:
        source = group_item['_source']
        task_name = source['task_name']
        submit_date = ts2datetime(int(source['submit_date']))
        submit_user = source['submit_user']
        detect_type = source['detect_type']
        state = source['state']
        process = source['detect_process']
        results.append(
            [task_name, submit_user, submit_date, detect_type, state, process])

    return results
Ejemplo n.º 16
0
def end_track_task(task_name):
    status = 0
    try:
        task_exist = es.get(index=index_name,
                            doc_type=index_type,
                            id=task_name)['_source']
    except:
        return 'task name not exist'
    task_status = task_exist['status']
    if status == '0':
        return 'task have end'
    else:
        task_exist['status'] = 0
        # made end time
        now_ts = time.time()
        now_date = ts2datetime(now_ts)
        now_date_ts = datetime2ts(now_date)
        time_segment = int((now_ts - now_date_ts) / 900) + 1
        end_ts = now_date_ts + time_segment * 900
        end_date = ts2date(end_ts)
        task_exist['end_date'] = end_date
        task_user = task_exist['uid_list']
        status = change_user_count(task_user)
        if status == 0:
            return 'change user task count fail'
        else:
            es.index(index=index_name,
                     doc_type=index_type,
                     id=task_name,
                     body=task_exist)
            status = delete_task_redis(task_name)
            if status == 0:
                return 'delete task from redis fail'
            else:
                return 'success change status to end'
def end_track_task(task_name):
    status = 0
    try:
        task_exist = es.get(index=index_name, doc_type=index_type, id=task_name)['_source']
    except:
        return 'task name not exist'
    task_status = task_exist['status']
    if status == '0':
        return 'task have end'
    else:
        task_exist['status'] = 0
        # made end time
        now_ts = time.time()
        now_date = ts2datetime(now_ts)
        now_date_ts = datetime2ts(now_date)
        time_segment = int((now_ts - now_date_ts) / 900) + 1
        end_ts = now_date_ts + time_segment * 900
        end_date = ts2date(end_ts)
        task_exist['end_date'] = end_date
        task_user = task_exist['uid_list']
        status = change_user_count(task_user)
        if status == 0:
            return 'change user task count fail'
        else:
            es.index(index=index_name, doc_type=index_type, id=task_name, body=task_exist)
            status = delete_task_redis(task_name)
            if status == 0:
                return 'delete task from redis fail'
            else:
                return 'success change status to end'
Ejemplo n.º 18
0
def submit_attribute(attribute_name, attribute_value, submit_user,
                     submit_date):
    status = False
    #maybe there have to identify the user admitted to submit attribute
    try:
        attribute_exist = es.get(index=attribute_index_name,
                                 doc_type=attribute_index_type,
                                 id=attribute_name)['docs']
    except:
        attribute_exist = {}
    try:
        source = attribute_exist['_source']
    except:
        input_data = dict()
        now_ts = time.time()
        date = ts2datetime(now_ts)
        input_data['attribute_name'] = attribute_name
        input_data['attribute_value'] = '&'.join(attribute_value.split(','))
        input_data['user'] = submit_user
        input_data['date'] = submit_date
        es.index(index=attribute_index_name,
                 doc_type=attribute_index_type,
                 id=attribute_name,
                 body=input_data)
        status = True
    return status
Ejemplo n.º 19
0
def get_user_hashtag(uid):
    user_hashtag_dict = {}
    sensitive_user_hashtag_dict = {}
    now_ts = time.time()
    now_date = ts2datetime(now_ts)  # 2015-09-22
    ts = datetime2ts(now_date)

    #test
    ts = datetime2ts('2013-09-08')
    for i in range(1, 8):
        ts = ts - 3600 * 24
        date = ts2datetime(ts).replace('-', '')
        results = r_cluster.hget('hashtag_' + str(date), uid)
        sensitive_results = r_cluster.hget('sensitive_hashtag_' + str(date),
                                           uid)
        if results:
            hashtag_dict = json.loads(results)
            for hashtag in hashtag_dict:
                if user_hashtag_dict.has_key(hashtag):
                    user_hashtag_dict[hashtag] += hashtag_dict[hashtag]
                else:
                    user_hashtag_dict[hashtag] = hashtag_dict[hashtag]
        if sensitive_results:
            sensitive_hashtag_dict = json.loads(sensitive_results)
            for hashtag in sensitive_hashtag_dict:
                if sensitive_user_hashtag_dict.has_key(hashtag):
                    sensitive_user_hashtag_dict[
                        hashtag] += sensitive_hashtag_dict[hashtag]
                else:
                    sensitive_user_hashtag_dict[
                        hashtag] = sensitive_hashtag_dict[hashtag]
    ordinary_key_set = set(user_hashtag_dict.keys())
    sensitive_key_set = set(sensitive_user_hashtag_dict.keys())
    for key in sensitive_key_set:
        if key in ordinary_key_set:
            user_hashtag_dict[key] += sensitive_user_hashtag_dict[key]
        else:
            user_hashtag_dict[key] = sensitive_user_hashtag_dict[key]

    sort_hashtag_dict = sorted(user_hashtag_dict.items(),
                               key=lambda x: x[1],
                               reverse=True)
    sort_sensitive_dict = sorted(sensitive_user_hashtag_dict.items(),
                                 key=lambda x: x[1],
                                 reverse=True)
    return [sort_hashtag_dict, sort_sensitive_dict]
Ejemplo n.º 20
0
def sort_sensitive_text(uid):
    sensitive_text = search_sensitive_text(uid)
    text_all = []
    if sensitive_text:
        for item in sensitive_text:
            text_detail = []
            item = item["_source"]
            if not item["sensitive"]:
                continue
            text = item["text"].encode("utf-8", "ignore")
            sentiment_dict = json.loads(item["sentiment"])
            if not sentiment_dict:
                sentiment = 0
            else:
                positive = len(sentiment_dict.get("126", {}))
                negetive = (
                    len(sentiment_dict.get("127", {}))
                    + len(sentiment_dict.get("128", {}))
                    + len(sentiment_dict.get("129", {}))
                )
                if positive > negetive:
                    sentiment = 1
                elif positive < negetive:
                    sentiment = -1
                else:
                    sentiment = 0
            ts = item["timestamp"]
            uid = item["uid"]
            mid = item["mid"]
            message_type = item.get("message_type", 0)
            date = ts2datetime(float(ts)).replace("-", "")
            try:
                bci_result = es.get(index=date, doc_type="bci", id=uid)["_source"]
                if int(message_type) == 1:
                    retweeted_number = bci_result["s_origin_weibo_retweeted_detail"].get(mid)
                    comment_number = bci_result["s_origin_weibo_comment_detail"].get(mid)
                elif int(message_type) == 2:
                    retweeted_number = bci_result["s_retweeted_weibo_retweeted_detail"].get(mid)
                    comment_number = bci_result["s_retweeted_weibo_comment_detail"].get(mid)
                else:
                    retweeted_number = 0
                    comment_number = 0
            except:
                retweeted_number = 0
                comment_number = 0
            single_sw = item.get("sensitive_words", {})
            if single_sw:
                sw = json.loads(single_sw).keys()
            else:
                # print item
                sw = []
            geo = item["geo"]
            retweeted_link = extract_uname(text)
            text_detail.extend(
                [ts, geo, text, sw, retweeted_link, sentiment, message_type, retweeted_number, comment_number]
            )
            text_all.append(text_detail)
    return text_all
Ejemplo n.º 21
0
def ajax_show_sensitive_history_in():
    results = []
    now_date = ts2datetime(time.time())
    date = request.args.get('date', now_date) # in date:2013-09-01
    if str(date) == "all":
        ts = time.time()
        now_ts = datetime2ts(now_date)
        for i in range(7):
            ts = now_ts - i*24*3600
            date = ts2datetime(ts)
            temp = show_in_history(date, 1)
            results.extend(temp)
    else:
        results = show_in_history(date, 1) # history in, include status
    if results:
        return json.dumps(results)
    else:
        return json.dumps([])
def get_text_index(date):
    now_ts = datetime2ts(date)
    index_list = []
    for i in range(7):
        ts = now_ts - i*DAY
        tmp_index = pre_text_index + ts2datetime(ts)
        index_list.append(tmp_index)

    return index_list
Ejemplo n.º 23
0
def get_activity_weibo(task_name,
                       submit_user,
                       start_ts,
                       time_segment=FOUR_HOUR):
    results = []
    #step1: get task_name uid
    task_id = submit_user + task_name
    try:
        group_result = es_group_result.get(index=group_index_name, doc_type=group_index_type ,\
                id=task_id, _source=False, fields=['uid_list'])
    except:
        group_result = {}
    if group_result == {}:
        return 'task name invalid'
    try:
        uid_list = group_result['fields']['uid_list']
    except:
        uid_list = []
    if uid_list == []:
        return 'task uid list null'
    #step2: get uid2uname
    uid2uname = {}
    try:
        user_portrait_result = es_user_portrait.mget(index=portrait_index_name, doc_type=portrait_index_type, \
                body = {'ids':uid_list}, _source=False, fields=['uname'])['docs']
    except:
        user_portrait_result = []
    for item in user_portrait_result:
        uid = item['_id']
        if item['found'] == True:
            uname = item['fields']['uname'][0]
        uid2uname[uid] = uname
    #step3: search time_segment weibo
    end_ts = start_ts + time_segment
    time_date = ts2datetime(start_ts)
    flow_text_index_name = flow_text_index_name_pre + time_date
    query = []
    query.append({'terms': {'uid': uid_list}})
    query.append({'range': {'timestamp': {'gte': start_ts, 'lt': end_ts}}})
    try:
        flow_text_es_result = es_flow_text.search(index=flow_text_index_name, doc_type=flow_text_index_type, \
                body={'query':{'bool':{'must':query}}, 'sort':'timestamp', 'size':MAX_VALUE})['hits']['hits']
    except:
        flow_text_es_result = []
    for item in flow_text_es_result:
        weibo = {}
        source = item['_source']
        weibo['timestamp'] = ts2date(source['timestamp'])
        weibo['ip'] = source['ip']
        weibo['text'] = source['text']
        if source['geo']:
            weibo['geo'] = '\t'.join(source['geo'])
        else:
            weibo['geo'] = ''
        results.append(weibo)

    return results
Ejemplo n.º 24
0
def ajax_show_sensitive_history_in():
    results = []
    now_date = ts2datetime(time.time())
    date = request.args.get('date', now_date) # in date:2013-09-01
    if str(date) == "all":
        ts = time.time()
        now_ts = datetime2ts(now_date)
        for i in range(7):
            ts = now_ts - i*24*3600
            date = ts2datetime(ts)
            temp = show_in_history(date, 1)
            results.extend(temp)
    else:
        results = show_in_history(date, 1) # history in, include status
    if results:
        return json.dumps(results)
    else:
        return json.dumps([])
Ejemplo n.º 25
0
def ajax_show_influence_history_in():
    results = []
    now_date = ts2datetime(time.time())
    date = request.args.get('date', now_date)
    if str(date) == "all":
        ts = time.time()
        now_ts = datetime2ts('2013-09-07')
        for i in range(7):
            ts = now_ts - i*24*3600
            date = ts2datetime(ts)
            date = str(date).replace('-', '')
            temp = show_in_history(date, 1)
            results.extend(temp)
    else:
        date = str(date).replace('-','')
        results = show_in_history(date, 0) # history in, include status
    if results:
        return json.dumps(results)
    else:
        return json.dumps([])
Ejemplo n.º 26
0
def ajax_show_influence_history_in():
    results = []
    now_date = ts2datetime(time.time())
    date = request.args.get('date', now_date)
    if str(date) == "all":
        ts = time.time()
        now_ts = datetime2ts('2013-09-07')
        for i in range(7):
            ts = now_ts - i * 24 * 3600
            date = ts2datetime(ts)
            date = str(date).replace('-', '')
            temp = show_in_history(date, 1)
            results.extend(temp)
    else:
        date = str(date).replace('-', '')
        results = show_in_history(date, 0)  # history in, include status
    if results:
        return json.dumps(results)
    else:
        return json.dumps([])
Ejemplo n.º 27
0
def show_in_history(date):
    print date
    results = []
    sensitive_uid_list = []
    influence_uid_list = []
    sen_iden_in_name = "identify_in_sensitive_" + str(date)
    inf_iden_in_name = "identify_in_influence_" + str(date)
    man_iden_in_name = "identify_in_manual_" + str(date)
    sen_iden_in_results = r.hgetall(sen_iden_in_name)
    inf_iden_in_results = r.hgetall(inf_iden_in_name)
    man_iden_in_results = r.hgetall(man_iden_in_name)
    sensitive_uid_list = sen_iden_in_results.keys()
    influence_uid_list = inf_iden_in_results.keys()
    manual_uid_list = man_iden_in_results.keys()
    #compute_results = r.hgetall('compute')
    results = []
    work_date = ts2datetime(datetime2ts(date) - DAY)

    if sensitive_uid_list:
        sensitive_results = get_sensitive_user_detail(sensitive_uid_list,
                                                      work_date, 1)
    else:
        sensitive_results = []
    for item in sensitive_results:
        uid = item[0]
        status = sen_iden_in_results[uid]
        item.append(status)
        results.append(item)

    if influence_uid_list:
        influence_results = get_sensitive_user_detail(influence_uid_list,
                                                      work_date, 0)
    else:
        influence_results = []
    for item in influence_results:
        uid = item[0]
        status = inf_iden_in_results[uid]
        item.append(status)
        results.append(item)

    if manual_uid_list:
        manual_results = get_sensitive_user_detail(manual_uid_list, work_date,
                                                   0)
    else:
        manual_results = []
    for item in manual_results:
        uid = item[0]
        status = man_iden_in_results[uid]
        item.append(status)
        results.append(item)

    sorted_results = sorted(results, key=lambda x: x[5], reverse=True)
    return sorted_results
def search_detect_task(task_name, submit_date, state, process, detect_type, submit_user):
    results = []
    query = [{'match':{'task_type': 'detect'}}]
    condition_num = 0
    if task_name:
        task_name_list = task_name.split(' ')
        for item in task_name_list:
            query.append({'wildcard':{'task_name': '*'+item+'*'}})
            condition_num += 1
    if submit_date:
        submit_date_ts = datetime2ts(submit_date)
        submit_date_from = submit_date_ts
        submit_date_to = submit_date_ts + DAY
        query.append({'range':{'submit_date':{'gte':submit_date_from, 'lt':submit_date_to}}})
        condition_num += 1
    if state:
        state_list = state.split(' ')
        for item in state_list:
            query.append({'wildcard':{'state': '*'+item+'*'}})
            condition_num += 1
    if process:
        query.append({'range':{'detect_process':{'from': int(process), 'to': MAX_PROCESS}}})
        condition_num += 1
    if detect_type:
        
        detect_type_list = detect_type.split(',')
        nest_body_list = []
        for type_item in detect_type_list:
            nest_body_list.append({'wildcard':{'detect_type': '*'+type_item+'*'}})
        query.append({'bool':{'should': nest_body_list}})
        
        condition_num += 1
    if submit_user:
        query.append({'wildcard':{'submit_user': '******'+submit_user+'*'}})
        condition_num += 1
    try:
        search_result = es_group_result.search(index=group_index_name, doc_type=group_index_type, \
                    body={'query':{'bool': {'must': query}}, 'sort':[{'submit_date': {'order': 'desc'}}], 'size':MAX_VALUE})['hits']['hits']
    except:
        search_result = []
    #get group information table
    for group_item in search_result:
        source = group_item['_source']
        task_name = source['task_name']
        submit_date = ts2datetime(int(source['submit_date']))
        submit_user = source['submit_user']
        detect_type = source['detect_type']
        state = source['state']
        process = source['detect_process']

        results.append([task_name, submit_user, submit_date, detect_type, state, process])
        
    return results
Ejemplo n.º 29
0
def lastest_identify_in():
    results = dict()
    now_ts = time.time()
    now_ts = datetime2ts('2013-09-08')
    for i in range(1,8):
        ts = now_ts - i * 3600 *24
        date = ts2datetime(ts).replace('-','')
        words_dict = r.hgetall('history_in_'+date)
        for item in words_dict:
            results[item] = json.loads(words_dict[item])

    return results
def sort_sensitive_text(uid):
    sensitive_text = search_sensitive_text(uid)
    text_all = []
    if sensitive_text:
        for item in sensitive_text:
            text_detail = []
            item = item['_source']
            if not item['sensitive']:
                continue
            text = item['text'].encode('utf-8', 'ignore')
            sentiment_dict = json.loads(item['sentiment'])
            if not sentiment_dict:
                sentiment = 0
            else:
                positive = len(sentiment_dict.get('126', {}))
                negetive = len(sentiment_dict.get('127', {})) + len(sentiment_dict.get('128', {})) + len(sentiment_dict.get('129', {}))
                if positive > negetive:
                    sentiment = 1
                elif positive < negetive:
                    sentiment = -1
                else:
                    sentiment = 0
            ts =item['timestamp']
            uid = item['uid']
            mid = item['mid']
            message_type = item.get('message_type', 0)
            date = ts2datetime(float(ts)).replace('-', '')
            try:
                bci_result = es.get(index=date, doc_type='bci', id=uid)['_source']
                if int(message_type) == 1:
                    retweeted_number = bci_result['s_origin_weibo_retweeted_detail'].get(mid)
                    comment_number = bci_result['s_origin_weibo_comment_detail'].get(mid)
                elif int(message_type) == 2:
                    retweeted_number = bci_result['s_retweeted_weibo_retweeted_detail'].get(mid)
                    comment_number = bci_result['s_retweeted_weibo_comment_detail'].get(mid)
                else:
                    retweeted_number = 0
                    comment_number = 0
            except:
                retweeted_number = 0
                comment_number = 0
            single_sw = item.get('sensitive_words', {})
            if single_sw:
                sw = json.loads(single_sw).keys()
            else:
                # print item
                sw = []
            geo = item['geo']
            retweeted_link = extract_uname(text)
            text_detail.extend([ts, geo, text, sw, retweeted_link, sentiment, message_type, retweeted_number, comment_number])
            text_all.append(text_detail)
    return text_all
def get_top_all_influence(key, ts):
    query_body = {
        "query":{
            "match_all": {}
        },
        "sort":{key:{"order":"desc"}},
        "size": 1
    }

    index_name = "bci_" + ts2datetime(ts).replace('-','')
    if not es.indices.exists(index=index_name):
        index_name = "bci_" + ts2datetime(ts-DAY).replace('-','')
    exist_es = es.indices.exists(index=index_name)
    if exist_es:
         search_result = es.search(index=index_name, doc_type="bci", body=query_body)['hits']['hits']
    else:
         search_result = {}
    if search_result:
        result = search_result[0]['_source'][key]
    else:
        result = 2000
    return result
Ejemplo n.º 32
0
def get_user_hashtag(uid):
    user_hashtag_dict = {}
    sensitive_user_hashtag_dict = {}
    now_ts = time.time()
    now_date = ts2datetime(now_ts) # 2015-09-22
    ts = datetime2ts(now_date)

    #test
    ts = datetime2ts('2013-09-08')
    for i in range(1,8):
        ts = ts - 3600*24
        date = ts2datetime(ts).replace('-','')
        results = r_cluster.hget('hashtag_'+str(date), uid)
        sensitive_results = r_cluster.hget('sensitive_hashtag_'+str(date), uid)
        if results:
            hashtag_dict = json.loads(results)
            for hashtag in hashtag_dict:
                if user_hashtag_dict.has_key(hashtag):
                    user_hashtag_dict[hashtag] += hashtag_dict[hashtag]
                else:
                    user_hashtag_dict[hashtag] = hashtag_dict[hashtag]
        if sensitive_results:
            sensitive_hashtag_dict = json.loads(sensitive_results)
            for hashtag in sensitive_hashtag_dict:
                if sensitive_user_hashtag_dict.has_key(hashtag):
                    sensitive_user_hashtag_dict[hashtag] += sensitive_hashtag_dict[hashtag]
                else:
                    sensitive_user_hashtag_dict[hashtag] = sensitive_hashtag_dict[hashtag]
    ordinary_key_set = set(user_hashtag_dict.keys())
    sensitive_key_set = set(sensitive_user_hashtag_dict.keys())
    for key in sensitive_key_set:
        if key in ordinary_key_set:
            user_hashtag_dict[key] += sensitive_user_hashtag_dict[key]
        else:
            user_hashtag_dict[key] = sensitive_user_hashtag_dict[key]

    sort_hashtag_dict = sorted(user_hashtag_dict.items(), key=lambda x:x[1], reverse=True)
    sort_sensitive_dict = sorted(sensitive_user_hashtag_dict.items(), key=lambda x:x[1], reverse=True)
    return [sort_hashtag_dict, sort_sensitive_dict]
Ejemplo n.º 33
0
def show_in_history(date):
    print date
    results = []
    sensitive_uid_list = []
    influence_uid_list = []
    sen_iden_in_name = "identify_in_sensitive_" + str(date)
    inf_iden_in_name = "identify_in_influence_" + str(date)
    man_iden_in_name = "identify_in_manual_" + str(date)
    sen_iden_in_results = r.hgetall(sen_iden_in_name)
    inf_iden_in_results = r.hgetall(inf_iden_in_name)
    man_iden_in_results = r.hgetall(man_iden_in_name)
    sensitive_uid_list = sen_iden_in_results.keys()
    influence_uid_list = inf_iden_in_results.keys()
    manual_uid_list = man_iden_in_results.keys()
    #compute_results = r.hgetall('compute')
    results = []
    work_date = ts2datetime(datetime2ts(date)-DAY)

    if sensitive_uid_list:
        sensitive_results = get_sensitive_user_detail(sensitive_uid_list, work_date, 1)
    else:
        sensitive_results = []
    for item in sensitive_results:
        uid = item[0]
        status = sen_iden_in_results[uid]
        item.append(status)
        results.append(item)

    if influence_uid_list:
        influence_results = get_sensitive_user_detail(influence_uid_list, work_date, 0)
    else:
        influence_results = []
    for item in influence_results:
        uid = item[0]
        status = inf_iden_in_results[uid]
        item.append(status)
        results.append(item)

    if manual_uid_list:
        manual_results = get_sensitive_user_detail(manual_uid_list, work_date, 0)
    else:
        manual_results = []
    for item in manual_results:
        uid = item[0]
        status = man_iden_in_results[uid]
        item.append(status)
        results.append(item)


    sorted_results = sorted(results, key=lambda x:x[5], reverse=True)
    return sorted_results
Ejemplo n.º 34
0
def user_sentiment_trend(uid):
    query_body = {"query": {"filtered": {"filter": {"term": {"uid": uid}}}}}
    search_results = es.search(index='sensitive_user_text',
                               doc_type='user',
                               body=query_body)['hits']['hits']
    sentiment_dict = dict()
    sentiment_results = dict()
    for item in search_results:
        datetime = ts2datetime(float(item['_source']['timestamp'])).replace(
            '-', '')
        try:
            sentiment_dict[datetime].append(
                json.loads(item['_source']['sentiment']))
        except:
            sentiment_dict[datetime] = [
                json.loads(item['_source']['sentiment'])
            ]
    total_positive = 0
    total_negetive = 0
    total_neutral = 0
    for datetime, sentiment_detail in sentiment_dict.items():
        positive_count = 0
        negetive_count = 0
        neutral_count = 0
        sentiment_results[datetime] = {}
        for item in sentiment_detail:
            if not item:
                try:
                    neutral_count += 1
                except:
                    neutral_count = 1
                total_neutral += 1
                continue
            positive_dict = item.get('126', {})
            positive = sum(positive_dict.values())
            positive_count += positive
            negetive = sum(item.get('127', {}).values()) + sum(
                item.get('128', {}).values()) + sum(
                    item.get('129', {}).values())
            negetive_count += negetive
            if positive > negetive:
                total_positive += 1
            elif positive < negetive:
                total_negetive += 1
            else:
                total_neutral += 1
        sentiment_results[datetime]['neutral'] = neutral_count
        sentiment_results[datetime]['positive'] = positive_count
        sentiment_results[datetime]['negetive'] = negetive_count
    return [[total_positive, total_neutral, total_negetive], sentiment_results]
def user_sentiment_trend(uid):
    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "term": {"uid": uid}
                }
            }
        }
    }
    search_results = es.search(index='sensitive_user_text', doc_type='user', body=query_body)['hits']['hits']
    sentiment_dict = dict()
    sentiment_results = dict()
    for item in search_results:
        datetime = ts2datetime(float(item['_source']['timestamp'])).replace('-', '')
        try:
            sentiment_dict[datetime].append(json.loads(item['_source']['sentiment']))
        except:
            sentiment_dict[datetime] = [json.loads(item['_source']['sentiment'])]
    total_positive = 0
    total_negetive = 0
    total_neutral = 0
    for datetime, sentiment_detail in sentiment_dict.items():
        positive_count = 0
        negetive_count = 0
        neutral_count = 0
        sentiment_results[datetime] = {}
        for item in sentiment_detail:
            if not item:
                try:
                    neutral_count += 1
                except:
                    neutral_count  = 1
                total_neutral += 1
                continue
            positive_dict = item.get('126', {})
            positive = sum(positive_dict.values())
            positive_count += positive
            negetive = sum(item.get('127', {}).values()) + sum(item.get('128', {}).values()) + sum(item.get('129', {}).values())
            negetive_count += negetive
            if positive > negetive:
                total_positive += 1
            elif positive < negetive:
                total_negetive += 1
            else:
                total_neutral += 1
        sentiment_results[datetime]['neutral'] = neutral_count
        sentiment_results[datetime]['positive'] = positive_count
        sentiment_results[datetime]['negetive'] = negetive_count
    return [[total_positive, total_neutral, total_negetive], sentiment_results]
Ejemplo n.º 36
0
def ajax_full_text_search():
    if RUN_TYPE:
        ts = time.time()
    else:
        ts = datetime2ts("2013-09-02")
    now_date = ts2datetime(ts)
    start_time = request.args.get("start_time", now_date)  # 2013-09-01
    end_time = request.args.get("end_time", now_date)
    uid = request.args.get("uid", "")
    size = request.args.get("number", 100)
    keywords = request.args.get("keywords", "")  # 逗号分隔

    results = full_text_search(keywords, uid, start_time, end_time, size)

    return json.dumps(results)
Ejemplo n.º 37
0
def recommend_in_sensitive(date):
    sensitive_name = "recomment_" + str(date) + "_sensitive"
    compute_name = "compute"
    re_sen_set = r.hkeys(sensitive_name)  # 敏感人物推荐
    iden_in_set = r.hkeys(compute_name)  # 已经入库用户
    if not re_sen_set:
        return []  # 那一天不存在数据
    uid_list = list(set(re_sen_set) - set(iden_in_set))
    sensitive = 1
    work_date = ts2datetime(datetime2ts(date) - DAY)
    if uid_list:
        results = get_sensitive_user_detail(uid_list, work_date, sensitive)
    else:
        results = []
    return results
def ajax_full_text_search():
    if RUN_TYPE:
        ts = time.time()
    else:
        ts = datetime2ts("2013-09-02")
    now_date = ts2datetime(ts)
    start_time = request.args.get("start_time", now_date) # 2013-09-01
    end_time = request.args.get("end_time", now_date)
    uid = request.args.get("uid", "")
    size = request.args.get("number", 100)
    keywords = request.args.get("keywords", "") # 逗号分隔

    results = full_text_search(keywords, uid, start_time, end_time, size)

    return json.dumps(results)
Ejemplo n.º 39
0
def recommend_in_sensitive(date):
    sensitive_name = "recomment_" + str(date) + "_sensitive"
    compute_name = "compute"
    re_sen_set = r.hkeys(sensitive_name) # 敏感人物推荐
    iden_in_set = r.hkeys(compute_name) # 已经入库用户
    if not re_sen_set:
        return [] # 那一天不存在数据
    uid_list = list(set(re_sen_set) - set(iden_in_set))
    sensitive = 1
    work_date = ts2datetime(datetime2ts(date)-DAY)
    if uid_list:
        results = get_sensitive_user_detail(uid_list, work_date, sensitive)
    else:
        results = []
    return results
Ejemplo n.º 40
0
def ajax_submit_task():
    input_data = dict()
    """
    input_data['task_name'] = request.args.get('task_name', '')
    input_data['uid_list'] = request.args.get('uid_list', '') # uid_list=[uid1, uid2]
    input_data['submit_date'] = request.args.get('submit_date', '')
    input_data['state'] = request.args.get('state', '')
    """
    input_data = request.get_json()
    #print input_data, type(input_data)
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    input_data['submit_date'] = now_date
    status = submit_task(input_data)
    return json.dumps(status)
Ejemplo n.º 41
0
def ajax_submit_task():
    input_data = dict()
    """
    input_data['task_name'] = request.args.get('task_name', '')
    input_data['uid_list'] = request.args.get('uid_list', '') # uid_list=[uid1, uid2]
    input_data['submit_date'] = request.args.get('submit_date', '')
    input_data['state'] = request.args.get('state', '')
    """
    input_data = request.get_json()
    #print input_data, type(input_data)
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    input_data['submit_date'] = now_date
    status = submit_task(input_data)
    return json.dumps(status)
Ejemplo n.º 42
0
def search_mention(uid, sensitive):
    date = ts2datetime(time.time()).replace('-', '')
    stat_results = dict()
    results = dict()
    test_ts = time.time()
    test_ts = datetime2ts('2013-09-07')
    for i in range(0, 7):
        ts = test_ts - i * 24 * 3600
        date = ts2datetime(ts).replace('-', '')
        if not sensitive:
            at_temp = r_cluster.hget('at_' + str(date), str(uid))
        else:
            at_temp = r_cluster.hget('sensitive_at_' + str(date), str(uid))
        if not at_temp:
            continue
        else:
            result_dict = json.loads(at_temp)
        for at_uid in result_dict:
            if stat_results.has_key(at_uid):
                stat_results[uid] += result_dict[at_uid]
            else:
                stat_results[uid] = result_dict[at_uid]
    if not stat_results:
        return [None, 0]

    in_status = identify_uid_list_in(result_dict.keys())
    for at_uid in result_dict:
        if at_uid in in_status:
            results[at_uid] = [result_dict[at_uid], '1']
        else:
            results[at_uid] = [result_dict[at_uid], '0']

    sorted_results = sorted(results.items(),
                            key=lambda x: x[1][0],
                            reverse=True)
    return [sorted_results[0:20], len(results)]
Ejemplo n.º 43
0
def ajax_task_sort():
    results = []
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    user = request.args.get('user', '')
    keyword = request.args.get("keyword", "") # 逗号分隔
    status = request.args.get("status", 2) # 2 for all, no limit
    start_time = request.args.get("start_time", "")
    end_time = request.args.get("end_time", now_date)
    submit_time = request.args.get('submit_time', "")
    status = int(status)
    
    #if user:
    results = sort_task(user, keyword, status, start_time, end_time, submit_time)

    return json.dumps(results)
Ejemplo n.º 44
0
def identify_in(date, words_list):
    # identify_in date and words_list(include level and category, [word, level, category])
    # date is date when new words were recommended
    ts = time.time()
    ts = datetime2ts('2013-09-07')
    time_list = []
    for i in range(7):
        now_ts = int(ts) - i*24*3600
        now_date = ts2datetime(now_ts).replace('-', '')
        time_list.append(now_date)
    for item in words_list:
        r.hset('sensitive_words', item[0], json.dumps([item[1], item[2]]))
        r.hset('history_in_'+date, item[0], json.dumps([item[1], item[2]]))
        for date in time_list:
            r.hdel('recommend_sensitive_words_'+date, item[0])
    return '1'
Ejemplo n.º 45
0
def ajax_task_sort():
    results = []
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    user = request.args.get('user', '')
    keyword = request.args.get("keyword", "") # 逗号分隔
    status = request.args.get("status", 2) # 2 for all, no limit
    start_time = request.args.get("start_time", "")
    end_time = request.args.get("end_time", now_date)
    submit_time = request.args.get('submit_time', "")
    status = int(status)
    
    if user:
        results = sort_task(user, keyword, status, start_time, end_time, submit_time)

    return json.dumps(results)
Ejemplo n.º 46
0
def user_sentiment_trend(uid):
    query_body = {"query": {"filtered": {"filter": {"term": {"uid": uid}}}}}
    search_results = es.search(index="sensitive_user_text", doc_type="user", body=query_body)["hits"]["hits"]
    sentiment_dict = dict()
    sentiment_results = dict()
    for item in search_results:
        datetime = ts2datetime(float(item["_source"]["timestamp"])).replace("-", "")
        try:
            sentiment_dict[datetime].append(json.loads(item["_source"]["sentiment"]))
        except:
            sentiment_dict[datetime] = [json.loads(item["_source"]["sentiment"])]
    total_positive = 0
    total_negetive = 0
    total_neutral = 0
    for datetime, sentiment_detail in sentiment_dict.items():
        positive_count = 0
        negetive_count = 0
        neutral_count = 0
        sentiment_results[datetime] = {}
        for item in sentiment_detail:
            if not item:
                try:
                    neutral_count += 1
                except:
                    neutral_count = 1
                total_neutral += 1
                continue
            positive_dict = item.get("126", {})
            positive = sum(positive_dict.values())
            positive_count += positive
            negetive = (
                sum(item.get("127", {}).values())
                + sum(item.get("128", {}).values())
                + sum(item.get("129", {}).values())
            )
            negetive_count += negetive
            if positive > negetive:
                total_positive += 1
            elif positive < negetive:
                total_negetive += 1
            else:
                total_neutral += 1
        sentiment_results[datetime]["neutral"] = neutral_count
        sentiment_results[datetime]["positive"] = positive_count
        sentiment_results[datetime]["negetive"] = negetive_count
    return [[total_positive, total_neutral, total_negetive], sentiment_results]
Ejemplo n.º 47
0
def recommend_in_top_influence(date):
    influence_name = "recomment_" + date + "_influence"
    identify_in_name = "compute"
    re_inf_set = r.hkeys(influence_name)
    iden_in_set = r.hkeys(identify_in_name) # 已经入库用户

    if not re_inf_set:
        return []
    else:
        uid_list = list(set(re_inf_set) - set(iden_in_set))
    sensitive = 0
    work_date = ts2datetime(datetime2ts(date)-DAY)
    if uid_list:
        results = get_sensitive_user_detail(uid_list, work_date, sensitive)
    else:
        results = []
    return results
Ejemplo n.º 48
0
def recommend_in_top_influence(date):
    influence_name = "recomment_" + date + "_influence"
    identify_in_name = "compute"
    re_inf_set = r.hkeys(influence_name)
    iden_in_set = r.hkeys(identify_in_name)  # 已经入库用户

    if not re_inf_set:
        return []
    else:
        uid_list = list(set(re_inf_set) - set(iden_in_set))
    sensitive = 0
    work_date = ts2datetime(datetime2ts(date) - DAY)
    if uid_list:
        results = get_sensitive_user_detail(uid_list, work_date, sensitive)
    else:
        results = []
    return results
Ejemplo n.º 49
0
def change_attribute(attribute_name, value, user, state):
    status = False
    # identify the attribute_name is in ES - custom attribute
    try:
        result =  es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
    except:
        result = None
        return status
    value_list = '&'.join(value.split(','))
    result['attribute_name'] = attribute_name
    result['attribute_value'] = value_list
    result['user'] = user
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    result['date'] = now_date
    es.index(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name ,body=result)
    status = True
    return status
Ejemplo n.º 50
0
def change_attribute(attribute_name, value, user, state):
    status = False
    # identify the attribute_name is in ES - custom attribute
    try:
        result =  es.get(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)['_source']
    except:
        result = None
        return status
    value_list = '&'.join(value.split(','))
    result['attribute_name'] = attribute_name
    result['attribute_value'] = value_list
    result['user'] = user
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    result['date'] = now_date
    es.index(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name ,body=result)
    status = True
    return status
Ejemplo n.º 51
0
def upload_file():
    upload_data = request.form['upload_data']
    task_name = request.form['task_name']
    state = request.form['state']
    now_ts = time.time()
    now_date = ts2datetime(now_ts)
    line_list = upload_data.split('\n')
    input_data = {}
    input_data['submit_date'] = now_date
    input_data['task_name'] = task_name
    input_data['state'] = state
    uid_list = []
    for line in line_list:
        uid = line[:10]
        if len(uid) == 10:
            uid_list.append(uid)
    input_data['uid_list'] = uid_list
    status = submit_task(input_data)
    return json.dumps(status)
Ejemplo n.º 52
0
def submit_attribute(attribute_name, attribute_value, submit_user, submit_date):
    status = False
    #maybe there have to identify the user admitted to submit attribute
    exist_bool = es.exists(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name)
    if exist_bool:
        return "tag exists"
    else:
        input_data = dict()
        now_ts = time.time()
        date = ts2datetime(now_ts)
        input_data['attribute_name'] = attribute_name
        input_data['attribute_value'] = '&'.join(attribute_value.split(','))
        input_data['user'] = submit_user
        input_data['date'] = submit_date
        es.index(index=attribute_index_name, doc_type=attribute_index_type, id=attribute_name, body=input_data)

        submit_tag = "tag-" + attribute_name
        exist_field = es_user_portrait.indices.get_field_mapping(index=user_index_name, doc_type=user_index_type, field=submit_tag)
        if not exist_field:
            print es_user_portrait.indices.put_mapping(index=user_index_name, doc_type=user_index_type,body={'properties':{submit_tag:{'type':'string', 'analyzer':'my_analyzer'}}}, ignore=400)
        status = True
    print status
    return status
def influenced_people(uid, mid, influence_style, date, default_number=20):
# uid 
# which weibo----mid, retweeted weibo ---seek for root_mid
# influence_style: retweeted(0) or comment(1)
    date1 = ts2datetime(datetime2ts(date)).replace('-', '')
    index_name = pre_index + date1
    index_flow_text = pre_text_index + date
    text_result = es.get(index=index_flow_text, doc_type=flow_text_index_type, id=mid)["_source"]
    temp_mid = text_result.get("root_mid",'') #判断微博是否是原创微博
    if temp_mid:
        mid_type = 1 # 非原创微博
    else:
        mid_type = 0 # 原创微博
    query_body = {
        "query":{
            "filtered":{
                "filter":{
                    "bool":{
                        "must":[
                        ]
                    }
                }
            }
        },
        "size": 30000
    }
    if RUN_TYPE:
        query_body["sort"] = {"user_fansnum":{"order":"desc"}}

    if int(mid_type) == 0:
        if int(influence_style) == 0: # origin weibo, all retweeted people
            query_body["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term": {"root_uid": uid}}, {"term": {"message_type": 3}}, {"term": {"root_mid": mid}}])
        else: # commented people
            query_body["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term": {"directed_uid": uid}}, {"term": {"message_type": 2}}, {"term": {"root_mid": mid}}])
    else:
        if int(influence_style) == 0: # origin weibo, all retweeted people
            query_body["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term": {"directed_uid": uid}}, {"term": {"message_type": 3}}, {"term": {"root_mid": temp_mid}}])
        else: # commented people
            query_body["query"]["filtered"]["filter"]["bool"]["must"].extend([{"term": {"directed_uid": uid}}, {"term": {"message_type": 2}}, {"term": {"root_mid": temp_mid}}])
    search_results = es.search(index=index_flow_text, doc_type=flow_text_index_type, body=query_body, _source=False, fields=["uid"], timeout=30)["hits"]["hits"]
    results = [] # uid_list
    if search_results:
        for item in search_results:
            if int(item["fields"]["uid"][0]) == int(uid):
                pass
            else:
                results.append(item["fields"]["uid"][0])
        results = list(set(results))
    else:
        results = []

    bci_index = "bci_" + date.replace('-','')

    if results:
        portrait_results = es_user_portrait.mget(index=user_portrait, doc_type=portrait_index_type, body={"ids": results}, fields=["domain", "topic_string", "activity_geo_dict","importance", "influence"])["docs"]
        bci_results = es_cluster.mget(index=bci_index, doc_type='bci', body={"ids":results}, fields=['user_index'])['docs']
    else:
        portrait_results = {}
        bci_results = {}


    in_portrait = []
    out_portrait = []
    in_portrait_info = []
    retweeted_domain = {}
    retweeted_topic = {}
    retweeted_geo = {}
    average_influence = 0
    total_influence = 0
    count = 0

    if bci_results:
        total_influence = 0
        for item in bci_results:
            if item['found']:
                total_influence += item['fields']['user_index'][0]
    try:
        average_influence = total_influence/len(results)
    except:
        average_influence = 0

    if portrait_results:
        for item in portrait_results:
            if item["found"]:
                temp = []
                count += 1
                temp.append(item['_id'])
                temp.append(item["fields"]["importance"][0])
                in_portrait.append(temp)
                temp_domain = item["fields"]["domain"][0].split('&')
                temp_topic = item["fields"]["topic_string"][0].split('&')
                temp_geo = json.loads(item["fields"]["activity_geo_dict"][0])[-1].keys()
                #total_influence += item["fields"]["influence"][0]
                retweeted_domain = aggregation(temp_domain, retweeted_domain)
                retweeted_topic = aggregation(temp_topic, retweeted_topic)
                retweeted_geo = aggregation(temp_geo, retweeted_geo)
            else:
                out_portrait.append(item['_id'])
        retweeted_domain = proportion(retweeted_domain)
        retweeted_topic = proportion(retweeted_topic)
        retweeted_geo = proportion(retweeted_geo)
        #try:
        #    average_influence = total_influence/count
        #except:
        #    average_influence = 0
    sorted_retweeted_domain = sorted(retweeted_domain.items(),key=lambda x:x[1], reverse=True)
    sorted_retweeted_topic = sorted(retweeted_topic.items(),key=lambda x:x[1], reverse=True)
    sorted_retweeted_geo = sorted(retweeted_geo.items(), key=lambda x:x[1], reverse=True)

    retweeted_results = dict()
    retweeted_results["domian"] = sorted_retweeted_domain[:5]
    retweeted_results["topic"] = sorted_retweeted_topic[:5]
    retweeted_results["geo"] = sorted_retweeted_geo[:5]
    retweeted_results["influence"] = average_influence
    in_portrait = sorted(in_portrait, key=lambda x:x[1], reverse=True)


    temp_list = []
    for item in in_portrait:
        temp_list.append(item[0])
    retweeted_results['in_portrait_number'] = len(temp_list)
    retweeted_results['out_portrait_number'] = len(out_portrait)
    in_portrait_url = get_user_url(temp_list[:default_number])
    out_portrait_url = get_user_url(out_portrait[:default_number])

    return_results = dict()
    return_results["influence_users"] = [in_portrait_url, out_portrait_url]
    return_results["influence_distribution"] = retweeted_results

    return return_results
Ejemplo n.º 54
0
def sensitive_attribute(uid, date):
    results = {}
    portrait = {}
    utype = user_type(uid)
    if not utype:
        results['utype'] = 0
        return results
    results['utype'] = 1

    results['uid'] = uid
    portrait_result = es.get(index='sensitive_user_portrait',
                             doc_type='user',
                             id=uid)['_source']
    results['uname'] = portrait_result['uname']
    if portrait_result['uname'] == 0:
        results['uname'] = 'unknown'
    if portrait_result['photo_url'] == 0:
        portrait_result['photo_url'] = 'unknown'
    if portrait_result['location'] == 0:
        portrait_result['location'] = 'unknown'
    results['photo_url'] = portrait_result['photo_url']

    # sensitive weibo number statistics
    date = ts2datetime(time.time() - 24 * 3600).replace('-', '')
    date = '20130907'  # test
    influence_results = []
    try:
        influence_results = es.get(index=date, doc_type='bci',
                                   id=uid)['_source']
        results['sensitive_origin_weibo_number'] = influence_results.get(
            's_origin_weibo_number', 0)
        results['sensitive_retweeted_weibo_number'] = influence_results.get(
            's_retweeted_weibo_number', 0)
        results['sensitive_comment_weibo_number'] = int(
            influence_results.get('s_comment_weibo_number', 0))
        results[
            'sensitive_retweeted_weibo_retweeted_total_number'] = influence_results.get(
                's_retweeted_weibo_retweeted_total_number', 0)
        results[
            'sensitive_origin_weibo_retweeted_total_number'] = influence_results.get(
                's_origin_weibo_retweeted_total_number', 0)
        results[
            'sensitive_origin_weibo_comment_total_number'] = influence_results.get(
                's_origin_weibo_comment_total_number', 0)
        results[
            'sensitive_retweeted_weibo_comment_total_number'] = influence_results.get(
                's_retweeted_weibo_comment_total_number', 0)
    except:
        results['sensitive_origin_weibo_number'] = 0
        results['sensitive_retweeted_weibo_number'] = 0
        results['sensitive_comment_weibo_number'] = 0
        results['sensitive_origin_weibo_retweeted_total_number'] = 0
        results['sensitive_origin_weibo_comment_total_number'] = 0
        results['sensitive_retweeted_weibo_retweeted_total_number'] = 0
        results['sensitive_retweeted_weibo_comment_total_number'] = 0

    try:
        item = es.get(index=date, doc_type='bci', id=uid)['_source']
    except:
        item = {}
    results['origin_weibo_total_number'] = item.get(
        'origin_weibo_number', 0) + results['sensitive_origin_weibo_number']
    results['retweeted_weibo_total_number'] = item.get(
        'retweeted_weibo_number',
        0) + results['sensitive_retweeted_weibo_number']
    results['comment_weibo_total_number'] = int(
        item.get('comment_weibo_number', 0)) + int(
            results['sensitive_comment_weibo_number'])
    results['origin_weibo_retweeted_total_number'] = item.get(
        'origin_weibo_retweeted_total_number',
        0) + results['sensitive_origin_weibo_retweeted_total_number']
    results['origin_weibo_comment_total_number'] = item.get(
        'origin_weibo_comment_total_number',
        0) + results['sensitive_origin_weibo_comment_total_number']
    results['retweeted_weibo_retweeted_total_number'] = item.get(
        'retweeted_weibo_retweeted_total_number',
        0) + results['sensitive_retweeted_weibo_retweeted_total_number']
    results['retweeted_weibo_comment_total_number'] = item.get(
        'retweeted_weibo_comment_total_number',
        0) + results['sensitive_retweeted_weibo_comment_total_number']

    results['sensitive_text'] = sort_sensitive_text(uid)

    results['sensitive_geo_distribute'] = []
    results['sensitive_time_distribute'] = get_user_trend(uid)[1]
    results['sensitive_hashtag'] = []
    results['sensitive_words'] = []
    results['sensitive_hashtag_dict'] = []
    results['sensitive_words_dict'] = []
    results['sensitive_hashtag_description'] = ''

    sentiment_trend = user_sentiment_trend(uid)
    emotion_number = sentiment_trend[0]
    results['negetive_index'] = float(emotion_number[2]) / (
        emotion_number[2] + emotion_number[1] + emotion_number[0])
    results['negetive_influence'] = float(emotion_number[1]) / (
        emotion_number[2] + emotion_number[1] + emotion_number[0])
    sentiment_dict = sentiment_trend[1]
    datetime = ts2datetime(time.time()).replace('-', '')
    return_sentiment = dict()
    return_sentiment['positive'] = []
    return_sentiment['neutral'] = []
    return_sentiment['negetive'] = []
    ts = time.time()
    ts = datetime2ts('2013-09-08') - 8 * 24 * 3600
    for i in range(1, 8):
        ts = ts + 24 * 3600
        date = ts2datetime(ts).replace('-', '')
        temp = sentiment_dict.get(date, {})
        return_sentiment['positive'].append([temp.get('positive', 0), date])
        return_sentiment['negetive'].append([temp.get('negetive', 0), date])
        return_sentiment['neutral'].append([temp.get('neutral', 0), date])
    results['sentiment_trend'] = return_sentiment

    if 1:
        portrait_results = es.get(index="sensitive_user_portrait",
                                  doc_type='user',
                                  id=uid)['_source']
        results['politics_trend'] = portrait_results['politics_trend']
        results['domain'] = portrait_results['domain']
        results['sensitive'] = portrait_results['sensitive']
        temp_hashtag = portrait_results['sensitive_hashtag_dict']
        temp_sensitive_words = portrait_results['sensitive_words_dict']
        temp_sensitive_geo = portrait_results['sensitive_geo_activity']
        if temp_sensitive_geo:
            sensitive_geo_dict = json.loads(temp_sensitive_geo)
            if len(sensitive_geo_dict) < 7:
                ts = time.time()
                ts = datetime2ts('2013-09-08') - 8 * 24 * 3600
                for i in range(7):
                    ts = ts + 24 * 3600
                    date = ts2datetime(ts).replace('-', '')
                    if sensitive_geo_dict.has_key(date):
                        pass
                    else:
                        sensitive_geo_dict[date] = {}
            sorted_sensitive_geo = sorted(sensitive_geo_dict.items(),
                                          key=lambda x: x[0],
                                          reverse=False)
            sensitive_geo_list = []
            for k, v in sorted_sensitive_geo:
                temp_list = []
                sorted_geo = sorted(v.items(),
                                    key=lambda x: x[1],
                                    reverse=True)[0:2]
                # print sorted_geo
                temp_list.extend([k, sorted_geo])
                sensitive_geo_list.append(temp_list)
            results['sensitive_geo_distribute'] = sensitive_geo_list
        if temp_hashtag:
            hashtag_dict = json.loads(
                portrait_results['sensitive_hashtag_dict'])
            if len(hashtag_dict) < 7:
                ts = time.time()
                ts = datetime2ts('2013-09-08') - 8 * 24 * 3600
                for i in range(7):
                    ts = ts + 24 * 3600
                    date = ts2datetime(ts).replace('-', '')
                    if hashtag_dict.has_key(date):
                        hashtag_dict_detail = hashtag_dict[date]
                        hashtag_dict[date] = sorted(
                            hashtag_dict_detail.items(),
                            key=lambda x: x[1],
                            reverse=True)
                    else:
                        hashtag_dict[date] = {}
            results['sensitive_hashtag_description'] = hashtag_description(
                hashtag_dict)
        else:
            hashtag_dict = {}
        if temp_sensitive_words:
            sensitive_words_dict = json.loads(temp_sensitive_words)
            if len(sensitive_words_dict) < 7:
                ts = time.time()
                ts = datetime2ts('2013-09-08') - 8 * 24 * 3600
                for i in range(7):
                    ts = ts + 24 * 3600
                    date = ts2datetime(ts).replace('-', '')
                    if sensitive_words_dict.has_key(date):
                        pass
                    else:
                        sensitive_words_dict[date] = {}
        else:
            sensitive_words_dict = {}
        date = ts2datetime(time.time() - 24 * 3600).replace('-', '')
        date = '20130907'
        today_sensitive_words = sensitive_words_dict.get(date, {})
        results['today_sensitive_words'] = today_sensitive_words
        all_hashtag_dict = {}
        for item in hashtag_dict:
            detail_hashtag_dict = hashtag_dict[item]
            for key in detail_hashtag_dict:
                if all_hashtag_dict.has_key(key[0]):
                    all_hashtag_dict[key[0]] += key[1]
                else:
                    all_hashtag_dict[key[0]] = key[1]

        all_sensitive_words_dict = {}
        for item in sensitive_words_dict:
            detail_words_dict = sensitive_words_dict[item]
            for key in detail_words_dict:
                if all_sensitive_words_dict.has_key(key):
                    all_sensitive_words_dict[key] += detail_words_dict[key]
                else:
                    all_sensitive_words_dict[key] = detail_words_dict[key]

        sorted_hashtag = sorted(all_hashtag_dict.items(),
                                key=lambda x: x[1],
                                reverse=True)
        sorted_words = sorted(all_sensitive_words_dict.items(),
                              key=lambda x: x[1],
                              reverse=True)
        sorted_hashtag_dict = sorted(hashtag_dict.items(),
                                     key=lambda x: x[0],
                                     reverse=False)
        sorted_words_dict = sorted(sensitive_words_dict.items(),
                                   key=lambda x: x[0],
                                   reverse=False)
        new_sorted_dict = sort_sensitive_words(sorted_words)
        results['sensitive_hashtag'] = sorted_hashtag
        results['sensitive_words'] = new_sorted_dict
        results['sensitive_hashtag_dict'] = sorted_hashtag_dict
        results['sensitive_words_dict'] = sorted_words_dict

    results['sensitive_retweet'] = search_retweet(uid, 1)
    results['sensitive_follow'] = search_follower(uid, 1)
    results['sensitive_at'] = search_mention(uid, 1)

    return results