def get_user_trend(uid): activity_dict = {} if RUN_TYPE: now_ts = time.time() now_date = ts2datetime(now_ts) # 2015-09-22 else: now_date = "2013-09-08" ts = datetime2ts(now_date) #test #ts = datetime2ts('2013-09-08') timestamp = ts return_results = dict() return_sensitive_results = {} for i in range(1, 8): ts = timestamp - 24 * 3600 * i date = ts2datetime(ts) if WORK_TYPE == 0: index_name = act_index_pre + date sensitive_index_name = sen_act_index_pre + date exist_bool = es_cluster.indices.exists(index=index_name) sensitive_exist_bool = es_cluster.indices.exists( index=sensitive_index_name) if exist_bool: try: tmp_act_result = es_cluster.get(index=index_name, doc_type="activity", id=uid)['_source'] results = tmp_act_result['activity_dict'] except: results = dict() else: results = dict() if sensitive_exist_bool: try: tmp_sensitive_act_result = es_cluster.get( index=sensitive_index_name, doc_type="sensitive_activity", id=uid)['_source'] sensitive_results = tmp_sensitive_ip_result[ 'sensitive_activity_dict'] except: sensitive_results = dict() else: sensitive_results = dict() else: results = redis_activity.hget('activity_' + str(ts), uid) sensitive_results = redis_activity.hget( 'sensitive_activity_' + str(ts), uid) if results: result_dict = json.loads(results) key_set = set(result_dict.keys()) for key in result_dict.keys(): return_results[int(key) * 900 + ts] = result_dict[key] else: pass if sensitive_results: sensitive_result_dict = json.loads(sensitive_results) for key in sensitive_result_dict.keys(): return_sensitive_results[int(key) * 900 + ts] = sensitive_result_dict[key] else: pass trend_dict = {} for i in range(1, 8): ts = timestamp - i * 24 * 3600 for j in range(0, 6): base_time = ts + j * 900 * 16 num = 0 for k in range(16): seg_time = base_time + k * 900 if seg_time in return_results: num += return_results[seg_time] trend_dict[base_time] = num sensitive_trend_dict = {} for i in range(1, 8): ts = timestamp - i * 24 * 3600 for j in range(0, 6): base_time = ts + j * 900 * 16 num = 0 for k in range(16): seg_time = base_time + k * 900 if seg_time in return_sensitive_results: num += return_sensitive_results[seg_time] sensitive_trend_dict[base_time] = num ordinary_key_set = set(trend_dict.keys()) sensitive_key_set = set(sensitive_trend_dict.keys()) for key in sensitive_key_set: if key in ordinary_key_set: trend_dict[key] += sensitive_trend_dict[key] else: trend_dict[key] = sensitive_trend_dict[key] sorted_dict = sorted(trend_dict.items(), key=lambda x: x[0], reverse=False) sorted_sensitive_dict = sorted(sensitive_trend_dict.items(), key=lambda x: x[0], reverse=False) return [sorted_dict, sorted_sensitive_dict] # total and sensitive
def get_user_trend(uid): activity_dict = {} if RUN_TYPE: now_ts = time.time() now_date = ts2datetime(now_ts) # 2015-09-22 else: now_date = "2013-09-08" ts = datetime2ts(now_date) #test #ts = datetime2ts('2013-09-08') timestamp = ts return_results = dict() return_sensitive_results = {} for i in range(1,8): ts = timestamp -24*3600*i date = ts2datetime(ts) if WORK_TYPE == 0: index_name = act_index_pre + date sensitive_index_name = sen_act_index_pre + date exist_bool = es_cluster.indices.exists(index=index_name) sensitive_exist_bool = es_cluster.indices.exists(index=sensitive_index_name) if exist_bool: try: tmp_act_result = es_cluster.get(index=index_name, doc_type="activity", id=uid)['_source'] results = tmp_act_result['activity_dict'] except: results = dict() else: results = dict() if sensitive_exist_bool: try: tmp_sensitive_act_result = es_cluster.get(index=sensitive_index_name, doc_type="sensitive_activity", id=uid)['_source'] sensitive_results = tmp_sensitive_ip_result['sensitive_activity_dict'] except: sensitive_results = dict() else: sensitive_results = dict() else: results = redis_activity.hget('activity_'+str(ts), uid) sensitive_results = redis_activity.hget('sensitive_activity_'+str(ts), uid) if results: result_dict = json.loads(results) key_set = set(result_dict.keys()) for key in result_dict.keys(): return_results[int(key)*900+ts] = result_dict[key] else: pass if sensitive_results: sensitive_result_dict = json.loads(sensitive_results) for key in sensitive_result_dict.keys(): return_sensitive_results[int(key)*900+ts] = sensitive_result_dict[key] else: pass trend_dict = {} for i in range(1,8): ts = timestamp - i*24*3600 for j in range(0,6): base_time = ts + j*900*16 num = 0 for k in range(16): seg_time = base_time + k*900 if seg_time in return_results: num += return_results[seg_time] trend_dict[base_time] = num sensitive_trend_dict = {} for i in range(1,8): ts = timestamp - i*24*3600 for j in range(0,6): base_time = ts + j*900*16 num = 0 for k in range(16): seg_time = base_time + k*900 if seg_time in return_sensitive_results: num += return_sensitive_results[seg_time] sensitive_trend_dict[base_time] = num ordinary_key_set = set(trend_dict.keys()) sensitive_key_set = set(sensitive_trend_dict.keys()) for key in sensitive_key_set: if key in ordinary_key_set: trend_dict[key] += sensitive_trend_dict[key] else: trend_dict[key] = sensitive_trend_dict[key] sorted_dict = sorted(trend_dict.items(), key=lambda x:x[0], reverse=False) sorted_sensitive_dict = sorted(sensitive_trend_dict.items(), key=lambda x:x[0], reverse=False) return [sorted_dict, sorted_sensitive_dict] # total and sensitive
def get_user_geo(uid): results = [] user_geo_result = {} user_ip_dict = {} user_ip_result = {} # ordinary ip user_sensitive_ip_result = {} # sensitive ip if RUN_TYPE: now_ts = time.time() now_date = ts2datetime(now_ts) # 2015-09-22 else: now_date = "2013-09-08" ts = datetime2ts(now_date) for i in range(1, 8): ts = ts - 3600 * 24 date = ts2datetime(ts) if WORK_TYPE == 0: index_name = ip_index_pre + str(date) sensitive_index_name = sen_ip_index_pre + str(date) exist_bool = es_cluster.indices.exists(index=index_name) sensitive_exist_bool = es_cluster.indices.exists( index=sensitive_index_name) if exist_bool: try: tmp_ip_result = es_cluster.get(index=index_name, doc_type="ip", id=uid)['_source'] results = tmp_ip_result['ip_dict'] except: results = dict() else: results = dict() if sensitive_exist_bool: try: tmp_sensitive_ip_result = es_cluster.get( index=sensitive_index_name, doc_type="sensitive_ip", id=uid)['_source'] sensitive_results = tmp_sensitive_ip_result[ 'sensitive_ip_dict'] except: sensitive_results = dict() else: sensitive_results = dict() else: results = redis_ip.hget('ip_' + str(ts), uid) sensitive_results = redis_ip.hget('sensitive_ip' + str(ts), uid) if results: ip_results = json.loads(results) for ip in ip_results: if user_ip_result.has_key(ip): user_ip_result[ip] += ip_results[ip] else: user_ip_result[ip] = ip_results[ip] if sensitive_results: sensitive_ip_results = json.loads(sensitive_results) for ip in sensitive_ip_results: if user_sensitive_ip_result.has_key(ip): user_sensitive_ip_result[ip] += sensitive_ip_results[ip] else: user_sensitive_ip_result[ip] = sensitive_ip_results[ip] ordinary_key_set = set(user_ip_result.keys()) sensitive_key_set = set(user_sensitive_ip_result.keys()) for key in sensitive_key_set: if key in ordinary_key_set: user_ip_result[key] += user_sensitive_ip_result[key] else: user_ip_result[key] = user_sensitive_ip_result[key] user_geo_dict = ip2geo(user_ip_result) sorted_user_geo_dict = sorted(user_geo_dict.items(), key=lambda x: x[1], reverse=True) sensitive_user_geo_dict = ip2geo(user_sensitive_ip_result) sorted_sensitive_user_geo_dict = sorted(sensitive_user_geo_dict.items(), key=lambda x: x[1], reverse=True) return_list = [] return_list = [sorted_user_geo_dict, sorted_sensitive_user_geo_dict] # total and sensitive return return_list
def get_user_geo(uid): results = [] user_geo_result = {} user_ip_dict = {} user_ip_result = {} # ordinary ip user_sensitive_ip_result = {} # sensitive ip if RUN_TYPE: now_ts = time.time() now_date = ts2datetime(now_ts) # 2015-09-22 else: now_date = "2013-09-08" ts = datetime2ts(now_date) for i in range(1,8): ts = ts - 3600*24 date = ts2datetime(ts) if WORK_TYPE == 0: index_name = ip_index_pre + str(date) sensitive_index_name = sen_ip_index_pre + str(date) exist_bool = es_cluster.indices.exists(index=index_name) sensitive_exist_bool = es_cluster.indices.exists(index=sensitive_index_name) if exist_bool: try: tmp_ip_result = es_cluster.get(index=index_name, doc_type="ip", id=uid)['_source'] results = tmp_ip_result['ip_dict'] except: results = dict() else: results = dict() if sensitive_exist_bool: try: tmp_sensitive_ip_result = es_cluster.get(index=sensitive_index_name, doc_type="sensitive_ip", id=uid)['_source'] sensitive_results = tmp_sensitive_ip_result['sensitive_ip_dict'] except: sensitive_results = dict() else: sensitive_results = dict() else: results = redis_ip.hget('ip_'+str(ts), uid) sensitive_results = redis_ip.hget('sensitive_ip'+str(ts), uid) if results: ip_results = json.loads(results) for ip in ip_results: if user_ip_result.has_key(ip): user_ip_result[ip] += ip_results[ip] else: user_ip_result[ip] = ip_results[ip] if sensitive_results: sensitive_ip_results = json.loads(sensitive_results) for ip in sensitive_ip_results: if user_sensitive_ip_result.has_key(ip): user_sensitive_ip_result[ip] += sensitive_ip_results[ip] else: user_sensitive_ip_result[ip] = sensitive_ip_results[ip] ordinary_key_set = set(user_ip_result.keys()) sensitive_key_set = set(user_sensitive_ip_result.keys()) for key in sensitive_key_set: if key in ordinary_key_set: user_ip_result[key] += user_sensitive_ip_result[key] else: user_ip_result[key] = user_sensitive_ip_result[key] user_geo_dict = ip2geo(user_ip_result) sorted_user_geo_dict = sorted(user_geo_dict.items(), key=lambda x:x[1], reverse=True) sensitive_user_geo_dict = ip2geo(user_sensitive_ip_result) sorted_sensitive_user_geo_dict = sorted(sensitive_user_geo_dict.items(), key=lambda x:x[1], reverse=True) return_list = [] return_list = [sorted_user_geo_dict, sorted_sensitive_user_geo_dict] # total and sensitive return return_list