def dispatch_by_province(self,time_range_data,country_dic,index):
'''
把time_range_data中的数据依次取出来,放到每个province中
:param time_range_data: :[[4, 1465715088.137754, u'127.0.0.1'], [5, 1465715202.913715, u'127.0.0.1']]
:param country_dic:
:param index: like "redirectTime" ...
:return:
'''
for data_point in time_range_data:
value,timestamp,ip_addr = data_point
ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_data_db)
lookup_res = ip_lookup_obj.lookpup(ip_addr)
if lookup_res:
print(lookup_res)
#print("%s :%s %s %s" %(ip_addr,lookup_res['province'],lookup_res['city'],lookup_res['district'] ))
for k,v in country_dic.items():
#print(v['name'],type(v['name']),lookup_res['province'],type(lookup_res['province']))
if v['name'].startswith(lookup_res['province']):
print("match:",v,v['name'])
data_point.append(lookup_res) #把解析结果也返回给前台
country_dic[k]['indexs'][index]['data'].append(data_point)
print("country dic data:",country_dic[k]['indexs'][index]['data'])
break
else:
print("\033[31;1m没找到匹配的省\033[0m")
def dispatch_realtime_region_watching(self):
'''
先检测有没有区域开启了实时监测,如果有,就对所有汇报过来的数据进行分析 ,把属于指定区域的数据放到相应的queue
:return:
'''
if self.region_realtime_watching_q: #如果不为空就代表 有用户开启了实时监控
client_ip = '123.133.157.27' #得到测试ip
ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_db_data)
lookup_res = ip_lookup_obj.lookpup(client_ip)
if lookup_res:
print('ip res:', lookup_res)
province_objs = models.Region.objects.filter(
name__startswith=lookup_res.get('province'))
city_objs = models.Region.objects.filter(
name__startswith=lookup_res.get('city'))
if province_objs and city_objs:
q_key = 'queue_%s_%s_%s' % (
self.site_obj.id, province_objs[0].id, city_objs[0].id)
if q_key in self.region_realtime_watching_q: #这个地区开启了实时分析 监测
print(
"region[%s][%s] has opened realtime watching....:"
% (province_objs[0].name, city_objs[0].name))
self.region_realtime_watching_q[q_key].put(
[self.data, client_ip])
def fetch_region_data(self, data_set, data_set_timestamp):
'''
把data_set里的值每个都遍历一次,对每个值的汇报ip做解析,如果是指定的地区的,就提取出来放 在一个列表里
:param data_set: [[25, 1466062380.695127, '61.184.33.228'], [28, 1466062381.782367, '124.23.180.0'], [29, 1466062383.278005, '125.72.101.82']]
:return:
'''
data_dic = {
'data': [],
'avg': None,
'mid': None,
'min': None,
'max': None,
'tp90': None,
'timestamp': data_set_timestamp
}
if data_set:
for data_point in data_set:
val, timestamp, ip_addr = data_point
ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_db_data)
lookup_res = ip_lookup_obj.lookpup(ip_addr)
if lookup_res:
print('ip res:', lookup_res)
province_objs = models.Region.objects.filter(
name__startswith=lookup_res.get('province'))
if province_objs:
if province_objs[0].id == int(
self.province_id): #是属于要监测的省的数据
print("match province:")
city_objs = models.Region.objects.filter(
name__startswith=lookup_res.get('city'))
if city_objs:
if city_objs[0].id == int(
self.city_id): #是属于要监测 的这个城市的
data_dic['data'].append(
data_point) #把这个数据加到要监测的地区统计列表里
print(
"\033[41;1mfind one match in region %s %s:\033[0m"
% (lookup_res['province'],
lookup_res['city']), data_point)
if data_dic['data']: #
only_value_list = [i[0] for i in data_dic['data']
] #把指标的值取出来交给下面的去计算平均值 什么的
data_dic['avg'] = self._get_avg(only_value_list)
data_dic['max'] = self._get_max(only_value_list)
data_dic['max'] = self._get_max(only_value_list)
data_dic['mid'] = self._get_mid(only_value_list)
data_dic['tp90'] = self._get_tp90(only_value_list)
return data_dic
def dispatch_by_city(self,time_range_data,city_dic,index): #把这些数据按地区维度进行分类清洗
'''
把time_range_data中的数据依次取出来,放到每个city中
:param time_range_data: [[4, 1465715088.137754, u'127.0.0.1'], [5, 1465715202.913715, u'127.0.0.1']]
:param city_dic:
:param index:
:return:
'''
for data_point in time_range_data:
value,timestamp,ip_addr = data_point
ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_data_db)
lookup_res = ip_lookup_obj.lookpup(ip_addr)
if lookup_res:
print(lookup_res)
#print("%s :%s %s %s" %(ip_addr,lookup_res['province'],lookup_res['city'],lookup_res['district'] ))
for k,v in city_dic.items():
#print(v['name'],type(v['name']),lookup_res['province'],type(lookup_res['province']))
if v['name'].startswith(lookup_res['city']):
print("match:",v,v['name'])
city_dic[k]['indexs'][index]['data'].append(data_point)
print("city dic data:",city_dic[k]['indexs'][index]['data'])
break
else:
print("\033[31;1m没找到匹配的地区\033[0m")
def __init__(self,redis_obj):
self.redis_obj = redis_obj
self.time_range = 900 #数据 分析时间范围
self.report_items = settings.REPORT_ITEMS
self.ip_data_db = ip_lookup.IPLookup(ip_db_filename=settings.IP_DB_FILE).ip_db_data
import json
from analyse.data_handler import DataProcess
from analyse.backends import redis_conn
from TengLan import settings
from analyse import models
from django.core.cache import cache
import queue
from analyse import data_handler
from analyse.backends import ip_lookup
REDIS_OBJ = redis_conn.redis_conn(settings)
GLOBAL_REALTIME_WATCHING_QUEUES = {} #如果前端页面打开了实时监测,所有指定区域来的数据到放到相应的Queue里
#print("loading ip db file".center(50,'-'))
IP_DB_DATA = ip_lookup.IPLookup(ip_db_filename=settings.IP_DB_FILE).ip_db_data
def data_report(request):
#print(request.POST)
#print(request.GET)
'''data = json.loads(request.GET.get('data'))
print(data)
if data.get('times') and data.get('resources_load_time'):
for key in data.get('times'):
print(key,data['times'][key])
print('resource load time'.center(40,'-'))
for k,v in data.get('resources_load_time').items():
print(k,v)
'''
data_process_obj = DataProcess(request, REDIS_OBJ,