def getCacheFileName(self, mo): span = self.cfgs.get('data_span', 5) * 60 loc = mo.getLocation() print mo.getId(), loc.time, timestamp_to_str(loc.time, fmt='%Y%m%d_%H%M%M') start = (loc.time // span) * span name = timestamp_to_str(start, fmt='%Y%m%d_%H%M') app_data_path = instance.getDataPath() path = os.path.join(app_data_path, self.cfgs.get('data_path'), name) if not os.path.exists(path): os.makedirs(path) filename = mo.getId().encode('utf-8') + '.txt' return os.path.join(path, filename)
def _time_twiste(self,loc): """时间扭曲不通过返回None""" enable = self.cfgs.get('timetwiste_enable',False) if not enable: return loc distance = self.cfgs.get('timetwiste_distance',60)*60 safe_time = time.time() - distance, time.time() if loc.time < safe_time[0] or loc.time > safe_time[1]: instance.getLogger().warning( u'<%s> getLocation error. vehicle:%s , gps 时间扭曲:%s' % (self.cfgs.get('name').upper(),loc.getId(), timestamp_to_str(loc.time))) self.time_twisters[loc.getId()] = { 'distance':int((time.time() - loc.time)/60.) , #记录扭曲时长 'sys_time':timestamp_to_str(time.time()), 'gps_time':timestamp_to_str(loc.time) } return None return loc
def _thread_persist(self): """ 将缓存数据持久化 1.数据校验,保证缓存的记录没有超出时间边界,并保证按时间排序 2.扫描时间从 1天前 - 距今两个缓存时间跨度 :return: """ seconds_oneday = 3600 * 24 span = self.cfgs.get('data_span') * 60 distance = self.cfgs.get('distance') * span app_data_path = instance.getDataPath() cache_path = os.path.join(app_data_path, self.cfgs.get('data_path')) back_path = os.path.join(app_data_path, self.cfgs.get('backup_path')) if not os.path.exists(back_path): os.makedirs(back_path) while True: now = time.time() end = (now // span) * span - distance start = end - seconds_oneday while start <= end: name = timestamp_to_str(start, fmt='%Y%m%d_%H%M') path = os.path.join(cache_path, name) # print path if not os.path.exists(path): start += span continue for _ in os.listdir(path): if _.find('.txt') == -1: continue self.processFile(start, span, os.path.join(path, _)) # 将数据目录移到备份目录 src = path dest = os.path.join(back_path, name) print 'src:', src print 'dest:', dest if os.path.exists(dest): shutil.rmtree(dest) shutil.move(src, dest) start += span gevent.sleep(5)
def write(self, start, span, datas): """写入车辆监控数据到pgsql :param datas :type datas:list (mos) """ if len(datas) == 0: return moid = datas[0].getId() cur = self.conn.cursor() table = "mo_data_" + timestamp_to_str(start, fmt='%Y%m%d') sql = "insert into {table} values(%s,%s,%s,%s)".format(table=table) list = [] print 'mo location:', moid, ' size:', len(datas) for ao in datas: list.append(ao.dict()) lastest = list[-1] # todo. 还需要检查轨迹点是否有效 try: cur.execute(sql, [moid, start, Json(list), Json(lastest)]) except: traceback.print_exc() instance.getLogger().warning(traceback.format_exc())
def _onDataReady(self, vehicle_id, data): """数据接收完成之后进行下一步消息的投递 """ data = self._time_twiste(data) if not data: return # return data data = self.buffer.enqueue(vehicle_id, data) if data: # fresh data env = data.toEnvelope() # 发送 DataEnvelope name = self.cfgs.get('post_mq') serial = env.marshall() print serial tick = timestamp_to_str(env.payloads.get( DataCategory.LOC.value).time, fmt='%Y%m%d_%H%M%S') print env.getId(), tick self.count += 1 print 'data sent:', self.count mq = AmqpManager.instance().getMessageQueue(name) mq.produce(serial)