def data_entry(data): try: Accumulator.instance().onDataRecieved(data) except: traceback.print_exc() instance.getLogger().warning(data) instance.getLogger().warning(traceback.format_exc())
def processFile(self, start, span, locfile): """ 排序、时间范围校验 :param start: :param span: :param locfile: :return: """ lines = open(locfile).readlines() lines = map(string.strip, lines) list = map(json.loads, lines) mos = [] for _ in list: mo = MovableObject.unmarshall(_) if not mo: continue loc = mo.getLocation() if loc.time < start or loc.time > start + span: instance.getLogger().warning( 'location time is not in (%s,%s)' % (start, start + span)) continue mos.append(mo) #根据时间排序 mos = sorted( mos, lambda x, y: cmp(x.getLocation().time, y.getLocation().time)) PersistenceManager.instance().write(start, span, mos)
def _request_getLocation(self, vehicles): """微线程执行 车辆位置信息的请求获取 :param vehicle_id: 车辆标识码 或者 车牌 :return: """ params = self._makeRequestParams_getLocation(vehicles) url = self.cfgs.get('access_url') resp = requests.post(url, data=params) content = resp.json() if content.get('code', -1) != 0: # 0: okay else false return records = content.get('data', []) self.dbg_req_num += len(vehicles) for data in records: code = data.get('code', -1) if code != 0: instance.getLogger().warning( '<G7> getLocation error. code:%s , vehicle:%s message:%s' % (data.get('code'), data.get('carnum', ''), data.get('message'))) if code == 9: # 车辆未绑定设备??? self.excluded.append(data.get('carnum', '')) continue vehicle_id = data.get('carnum') loc = LocationData(vehicle_id) # spawn location point loc.version = '0.1' loc.provider = ProviderType.G7.value loc.lon = data.get('lng', 0) loc.lat = data.get('lat', 0) loc.speed = data.get('speed', 0) loc.direction = data.get('course', 0) loc.status = 0 loc.address = data.get('address', '') loc.text = data.get('status', 0) loc.encode = LocationEncodeType.BD.value loc.time = str_to_timestamp(data.get('time', '')) # todo. format timestamp loc.extra['orgcode'] = data.get('orgcode', '') # 机构号 loc.extra['fromtype'] = data.get('fromtype', '') # 车辆类型 1为自建车辆,3为外部共享车辆 loc.extra['address'] = data.get('address', '') loc.extra['status'] = data.get('status', '') loc.extra['offline'] = data.get('offline', 0) # 离线持续时间(s) loc.extra['offline_start'] = data.get('movetime') # 离线开始时间 print loc.dict() print 'batch request num:', self.dbg_req_num self.dbg_resp_num += 1 print 'recieved locations num:', self.dbg_resp_num self._onDataReady(vehicle_id, loc)
def init(self, cfgs): self.cfgs = cfgs for cfg in self.cfgs: if not cfg.get('enable', False): continue cls = import_function(cfg.get('module')) adapter = cls(cfg) res = adapter.open() if res: self.registerAdapter(adapter.name, adapter) else: instance.getLogger().error('adapter %s open failed!' % adapter.name)
def _log(self,category,name,args): logger = instance.getLogger() root_cfg = instance.getConfig() level = root_cfg.get('level','INFO') text = '[Celery] categary:%s name:%s args:%s '%(category,name,args) text = text.replace('\n','') logger.log(level,text)
def _time_twiste(self,loc): """时间扭曲不通过返回None""" enable = self.cfgs.get('timetwiste_enable',False) if not enable: return loc distance = self.cfgs.get('timetwiste_distance',60)*60 safe_time = time.time() - distance, time.time() if loc.time < safe_time[0] or loc.time > safe_time[1]: instance.getLogger().warning( u'<%s> getLocation error. vehicle:%s , gps 时间扭曲:%s' % (self.cfgs.get('name').upper(),loc.getId(), timestamp_to_str(loc.time))) self.time_twisters[loc.getId()] = { 'distance':int((time.time() - loc.time)/60.) , #记录扭曲时长 'sys_time':timestamp_to_str(time.time()), 'gps_time':timestamp_to_str(loc.time) } return None return loc
def write(self, start, span, datas): """写入车辆监控数据到pgsql :param datas :type datas:list (mos) """ if len(datas) == 0: return moid = datas[0].getId() cur = self.conn.cursor() table = "mo_data_" + timestamp_to_str(start, fmt='%Y%m%d') sql = "insert into {table} values(%s,%s,%s,%s)".format(table=table) list = [] print 'mo location:', moid, ' size:', len(datas) for ao in datas: list.append(ao.dict()) lastest = list[-1] # todo. 还需要检查轨迹点是否有效 try: cur.execute(sql, [moid, start, Json(list), Json(lastest)]) except: traceback.print_exc() instance.getLogger().warning(traceback.format_exc())
def _auth(self): params = {'user': self.cfgs.get('account'), 'pwd': self.cfgs.get('password')} data = urllib.urlencode(params) data = crypto.encode(data) params={ 'client_id': self.cfgs.get('client_id') } url = self.cfgs.get('auth_url') + '/' + data # print 'auth request:'+ url cert = os.path.join(instance.getConfigPath(),self.cfgs.get('cert_file')) resp = requests.post(url,params,verify=cert) # print resp.text data = crypto.decode(resp.text) data = json.loads(data) if data.get('status') !=1001: instance.getLogger().warning('adapter(zjxl) auth request failed. status:%s'%data.get('status')) return False self.token = data.get('result') instance.getLogger().debug('adapter(zjxl) auth-token:'+ self.token)
def __request_getLocation(self,vehicles): self._auth() # for vehicle_id in vehicles: carnums = ','.join(vehicles) params = {'token': self.token, 'vclN': carnums.encode('utf-8'),'timeNearby':30 } data = urllib.urlencode(params) data = crypto.encode(data) url = self.cfgs.get('access_url_batch') +'/' + data params = {'client_id': self.cfgs.get('client_id')} cert = os.path.join(instance.getConfigPath(), self.cfgs.get('cert_file')) resp = requests.post(url, params, verify=cert) print resp.text data = crypto.decode(resp.text) data = json.loads(data) if data.get('status') != 1001: instance.getLogger().warning('adapter(zjxl) getLocation request failed. status:%s'%data.get('status')) return False resp = requests.post(url, data=params) content = resp.json() if content.get('code', -1) != 0: # 0: okay else false return data = content.get('result', {}) loc = LocationData(vehicle_id) # spawn location point loc.version = '0.1' loc.provider = ProviderType.ZJXL.value loc.lon = int(data.get('lng', 0))/600000.0 loc.lat = int(data.get('lat', 0))/600000.0 loc.speed = float(data.get('spd', 0)) loc.time = int(data.get('utc', 0)) # loc.direction = int( data.get('drc',0)) loc.extra['address'] = data.get('adr', '') self._onDataReady(vehicle_id, loc)
def _request_getLocation(self,vehicles): # return self.__request_getLocation(vehicles) self._auth() for vehicle_id in vehicles: # carnums = ','.join(vehicles) params = {'token': self.token, 'vclN': vehicle_id.encode('utf-8'),'timeNearby':30 } data = urllib.urlencode(params) data = crypto.encode(data) url = self.cfgs.get('access_url') +'/' + data params = {'client_id': self.cfgs.get('client_id')} cert = os.path.join(instance.getConfigPath(), self.cfgs.get('cert_file')) resp = requests.post(url, params, verify=cert) # print resp.text data = crypto.decode(resp.text) data = json.loads(data) if data.get('status') != 1001: instance.getLogger().warning('adapter(zjxl) getLocation request failed. status:%s'%data.get('status')) return False loc = self.assignLocation(vehicle_id,data) self._onDataReady(vehicle_id, loc)
def _request_getLocation(self, vehicles): """微线程执行 车辆位置信息的请求获取 :param vehicle_id: 车辆标识码 或者 车牌 :return: """ wsdl = self.cfgs.get('access_url') username = self.cfgs.get('account') sn_list = [] for _ in vehicles: plate, sn = map(string.strip, _.split('\t')) sn_list.append(sn) carnums = ','.join(sn_list) result = [] try: client = Client(wsdl=wsdl) result = client.service.get_running_info(username, carnums) result = json.loads(result) self.dbg_req_num += len(sn_list) except: instance.getLogger().warning('getLocation error. vehicle:%s ' % (carnums)) for data in result: if data.get('gpsbz', '') == '56': continue # 数据无效 vehicle_id = data.get('cph', '').strip() loc = LocationData(vehicle_id) # spawn location point loc.version = '0.1' loc.provider = ProviderType.ZQ.value loc.lon = data.get('jd', 0) loc.lat = data.get('wd', 0) loc.speed = data.get('cs', 0) loc.altitute = data.get('hb', 0) loc.time = str_to_timestamp(data.get('gpssj', 0)) # todo. format timestamp loc.text = u'设备号:%s,水温:%s,总里程:%s,总油耗:%s ' % (data.get( 'sbh', ''), data.get('sw', 0), data.get('zlc', 0), data.get('zyh', 0)) loc.status = 0 loc.address = data.get('addr', '') loc.direction = data.get('fx', 0) loc.extra['lisno'] = data.get('lisno', '') # 机构号 loc.extra['cjh'] = data.get('cjh', '') # 车辆类型 1为自建车辆,3为外部共享车辆 loc.extra['sbh'] = data.get('sbh', '') # 设备号 loc.extra['lx'] = data.get('lx', '') loc.extra['zs'] = data.get('zs', 0) # 离线持续时间(s) loc.extra['sw'] = data.get('sw', 0) # 水温 loc.extra['ssyh'] = data.get('ssyh', 0) # 瞬 时油耗 loc.extra['nj'] = data.get('nj', 0) # 瞬 时油耗 loc.extra['zlc'] = data.get('zlc', 0) # 总里程 loc.extra['zyh'] = data.get('zyh', 0) # 总油耗 loc.extra['ssyl'] = data.get('ssyl', 0) # 剩余油量 loc.extra['ym'] = data.get('ym', 0) # 油 门 loc.extra['dw'] = data.get('dw', 0) # 档位 loc.extra['fdjsj'] = data.get('fdjsj', 0) # 发动机工作时间 print loc.dict() print 'batch request num:', self.dbg_req_num self.dbg_resp_num += 1 print 'recieved locations num:', self.dbg_resp_num self._onDataReady(vehicle_id, loc)
def _request_getLocation(self,vehicles): """微线程执行 车辆位置信息的请求获取 :param vehicle_id: 车辆标识码 或者 车牌 :return: """ # print vehicles # gevent.sleep(0.01) params = self._makeRequestParams_getLocation(vehicles) url = self.cfgs.get('access_url') resp = requests.post(url, data=params) content = resp.json() result = content.get('result',{}) code = int(result.get('code',-1)) if code!=1: # 0: okay else false instance.getLogger().warning( 'getLocation error. code:%s , message:%s' % (code, result.get('message',''))) # return if code == 42: # 车辆未绑定设备??? # 1.多车查询时(接口:GetVehcileInfo),其中某一台车未绑定,导致此批次请求返回42错误。导致此批次其他车辆数据无法获取 # 2.请求频率0.2秒,也报ddos攻击 for _ in vehicles: self.excluded.append(_) return records = result.get('data',[]) self.dbg_req_num+=len(vehicles) print 'batch request num:',self.dbg_req_num self.dbg_resp_num+=len(records) print 'recieved locations num:',self.dbg_resp_num for data in records: vehicle_id = data.get('Vehicle') loc = LocationData( vehicle_id) # spawn location point loc.version = '0.1' loc.provider = ProviderType.YL.value loc.lon = data.get('Lon',0) loc.lat = data.get('Lat',0) loc.speed = data.get('Speed',0) loc.time = str_to_timestamp( data.get('GPSTime',0) ) # todo. format timestamp loc.direction = data.get('Direction',0) loc.encode = LocationEncodeType.BD.value loc.address = data.get('PlaceName','') + data.get('RoadName','') loc.text = data.get('Status','') loc.status = 0 loc.extra['Odometer'] = data.get('Odometer',0) # 里程 loc.extra['address'] = data.get('PlaceName','') loc.extra['status'] = data.get('Status','') #ACC开,3D定位,天线正常,冷机开,门关,门关4 loc.extra['Provice'] = data.get('Provice','') # 省 loc.extra['City'] = data.get('City','') # 市 loc.extra['District'] = data.get('District','') # 区 loc.extra['RoadName'] = data.get('RoadName','') # 路名信息 loc.extra['T1'] = data.get('T1','') # 温度1(℃) loc.extra['T2'] = data.get('T2','') # 温度2(℃) loc.extra['T3'] = data.get('T3','') # 温度3(℃) loc.extra['T4'] = data.get('T4','') # 温度4(℃) loc.extra['Lat02'] = data.get('Lat02','') # 根据参数isoffsetlonlat返回的纬度 loc.extra['Lon02'] = data.get('Lon02','') # 根据参数isoffsetlonlat返回的纬度 loc.extra['AreaName'] = data.get('AreaName','') # 地标名称 loc.extra['Time1'] = data.get('Time1','') # 温度1采集时间 2014-10-16 08:48:54 loc.extra['Time2'] = data.get('Time2','') # 温度1采集时间 2014-10-16 08:48:54 loc.extra['Time3'] = data.get('Time3','') # 温度1采集时间 2014-10-16 08:48:54 loc.extra['Time4'] = data.get('Time4','') # 温度1采集时间 2014-10-16 08:48:54 self._onDataReady( vehicle_id,loc )
def registerAdapter(self, name, adapter): self.adpaters[name] = adapter instance.getLogger().info('adapter: %s has been loaded..' % name)