def dispatch_by_province(self,time_range_data,country_dic,index): ''' 把time_range_data中的数据依次取出来,放到每个province中 :param time_range_data: :[[4, 1465715088.137754, u'127.0.0.1'], [5, 1465715202.913715, u'127.0.0.1']] :param country_dic: :param index: like "redirectTime" ... :return: ''' for data_point in time_range_data: value,timestamp,ip_addr = data_point ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_data_db) lookup_res = ip_lookup_obj.lookpup(ip_addr) if lookup_res: print(lookup_res) #print("%s :%s %s %s" %(ip_addr,lookup_res['province'],lookup_res['city'],lookup_res['district'] )) for k,v in country_dic.items(): #print(v['name'],type(v['name']),lookup_res['province'],type(lookup_res['province'])) if v['name'].startswith(lookup_res['province']): print("match:",v,v['name']) data_point.append(lookup_res) #把解析结果也返回给前台 country_dic[k]['indexs'][index]['data'].append(data_point) print("country dic data:",country_dic[k]['indexs'][index]['data']) break else: print("\033[31;1m没找到匹配的省\033[0m")
def dispatch_realtime_region_watching(self): ''' 先检测有没有区域开启了实时监测,如果有,就对所有汇报过来的数据进行分析 ,把属于指定区域的数据放到相应的queue :return: ''' if self.region_realtime_watching_q: #如果不为空就代表 有用户开启了实时监控 client_ip = '123.133.157.27' #得到测试ip ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_db_data) lookup_res = ip_lookup_obj.lookpup(client_ip) if lookup_res: print('ip res:', lookup_res) province_objs = models.Region.objects.filter( name__startswith=lookup_res.get('province')) city_objs = models.Region.objects.filter( name__startswith=lookup_res.get('city')) if province_objs and city_objs: q_key = 'queue_%s_%s_%s' % ( self.site_obj.id, province_objs[0].id, city_objs[0].id) if q_key in self.region_realtime_watching_q: #这个地区开启了实时分析 监测 print( "region[%s][%s] has opened realtime watching....:" % (province_objs[0].name, city_objs[0].name)) self.region_realtime_watching_q[q_key].put( [self.data, client_ip])
def fetch_region_data(self, data_set, data_set_timestamp): ''' 把data_set里的值每个都遍历一次,对每个值的汇报ip做解析,如果是指定的地区的,就提取出来放 在一个列表里 :param data_set: [[25, 1466062380.695127, '61.184.33.228'], [28, 1466062381.782367, '124.23.180.0'], [29, 1466062383.278005, '125.72.101.82']] :return: ''' data_dic = { 'data': [], 'avg': None, 'mid': None, 'min': None, 'max': None, 'tp90': None, 'timestamp': data_set_timestamp } if data_set: for data_point in data_set: val, timestamp, ip_addr = data_point ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_db_data) lookup_res = ip_lookup_obj.lookpup(ip_addr) if lookup_res: print('ip res:', lookup_res) province_objs = models.Region.objects.filter( name__startswith=lookup_res.get('province')) if province_objs: if province_objs[0].id == int( self.province_id): #是属于要监测的省的数据 print("match province:") city_objs = models.Region.objects.filter( name__startswith=lookup_res.get('city')) if city_objs: if city_objs[0].id == int( self.city_id): #是属于要监测 的这个城市的 data_dic['data'].append( data_point) #把这个数据加到要监测的地区统计列表里 print( "\033[41;1mfind one match in region %s %s:\033[0m" % (lookup_res['province'], lookup_res['city']), data_point) if data_dic['data']: # only_value_list = [i[0] for i in data_dic['data'] ] #把指标的值取出来交给下面的去计算平均值 什么的 data_dic['avg'] = self._get_avg(only_value_list) data_dic['max'] = self._get_max(only_value_list) data_dic['max'] = self._get_max(only_value_list) data_dic['mid'] = self._get_mid(only_value_list) data_dic['tp90'] = self._get_tp90(only_value_list) return data_dic
def dispatch_by_city(self,time_range_data,city_dic,index): #把这些数据按地区维度进行分类清洗 ''' 把time_range_data中的数据依次取出来,放到每个city中 :param time_range_data: [[4, 1465715088.137754, u'127.0.0.1'], [5, 1465715202.913715, u'127.0.0.1']] :param city_dic: :param index: :return: ''' for data_point in time_range_data: value,timestamp,ip_addr = data_point ip_lookup_obj = ip_lookup.IPLookup(ip_db_data=self.ip_data_db) lookup_res = ip_lookup_obj.lookpup(ip_addr) if lookup_res: print(lookup_res) #print("%s :%s %s %s" %(ip_addr,lookup_res['province'],lookup_res['city'],lookup_res['district'] )) for k,v in city_dic.items(): #print(v['name'],type(v['name']),lookup_res['province'],type(lookup_res['province'])) if v['name'].startswith(lookup_res['city']): print("match:",v,v['name']) city_dic[k]['indexs'][index]['data'].append(data_point) print("city dic data:",city_dic[k]['indexs'][index]['data']) break else: print("\033[31;1m没找到匹配的地区\033[0m")
def __init__(self,redis_obj): self.redis_obj = redis_obj self.time_range = 900 #数据 分析时间范围 self.report_items = settings.REPORT_ITEMS self.ip_data_db = ip_lookup.IPLookup(ip_db_filename=settings.IP_DB_FILE).ip_db_data
import json from analyse.data_handler import DataProcess from analyse.backends import redis_conn from TengLan import settings from analyse import models from django.core.cache import cache import queue from analyse import data_handler from analyse.backends import ip_lookup REDIS_OBJ = redis_conn.redis_conn(settings) GLOBAL_REALTIME_WATCHING_QUEUES = {} #如果前端页面打开了实时监测,所有指定区域来的数据到放到相应的Queue里 #print("loading ip db file".center(50,'-')) IP_DB_DATA = ip_lookup.IPLookup(ip_db_filename=settings.IP_DB_FILE).ip_db_data def data_report(request): #print(request.POST) #print(request.GET) '''data = json.loads(request.GET.get('data')) print(data) if data.get('times') and data.get('resources_load_time'): for key in data.get('times'): print(key,data['times'][key]) print('resource load time'.center(40,'-')) for k,v in data.get('resources_load_time').items(): print(k,v) ''' data_process_obj = DataProcess(request, REDIS_OBJ,