def getCacheFileName(self, mo):
span = self.cfgs.get('data_span', 5) * 60
loc = mo.getLocation()
print mo.getId(), loc.time, timestamp_to_str(loc.time,
fmt='%Y%m%d_%H%M%M')
start = (loc.time // span) * span
name = timestamp_to_str(start, fmt='%Y%m%d_%H%M')
app_data_path = instance.getDataPath()
path = os.path.join(app_data_path, self.cfgs.get('data_path'), name)
if not os.path.exists(path):
os.makedirs(path)
filename = mo.getId().encode('utf-8') + '.txt'
return os.path.join(path, filename)
def _time_twiste(self,loc):
"""时间扭曲不通过返回None"""
enable = self.cfgs.get('timetwiste_enable',False)
if not enable:
return loc
distance = self.cfgs.get('timetwiste_distance',60)*60
safe_time = time.time() - distance, time.time()
if loc.time < safe_time[0] or loc.time > safe_time[1]:
instance.getLogger().warning(
u'<%s> getLocation error. vehicle:%s , gps 时间扭曲:%s' %
(self.cfgs.get('name').upper(),loc.getId(), timestamp_to_str(loc.time)))
self.time_twisters[loc.getId()] = {
'distance':int((time.time() - loc.time)/60.) , #记录扭曲时长
'sys_time':timestamp_to_str(time.time()),
'gps_time':timestamp_to_str(loc.time)
}
return None
return loc
def _thread_persist(self):
"""
将缓存数据持久化
1.数据校验，保证缓存的记录没有超出时间边界，并保证按时间排序
2.扫描时间从 1天前 - 距今两个缓存时间跨度
:return:
"""
seconds_oneday = 3600 * 24
span = self.cfgs.get('data_span') * 60
distance = self.cfgs.get('distance') * span
app_data_path = instance.getDataPath()
cache_path = os.path.join(app_data_path, self.cfgs.get('data_path'))
back_path = os.path.join(app_data_path, self.cfgs.get('backup_path'))
if not os.path.exists(back_path):
os.makedirs(back_path)
while True:
now = time.time()
end = (now // span) * span - distance
start = end - seconds_oneday
while start <= end:
name = timestamp_to_str(start, fmt='%Y%m%d_%H%M')
path = os.path.join(cache_path, name)
# print path
if not os.path.exists(path):
start += span
continue
for _ in os.listdir(path):
if _.find('.txt') == -1:
continue
self.processFile(start, span, os.path.join(path, _))
# 将数据目录移到备份目录
src = path
dest = os.path.join(back_path, name)
print 'src:', src
print 'dest:', dest
if os.path.exists(dest):
shutil.rmtree(dest)
shutil.move(src, dest)
start += span
gevent.sleep(5)
def write(self, start, span, datas):
"""写入车辆监控数据到pgsql
:param datas
:type datas:list (mos)
"""
if len(datas) == 0:
return
moid = datas[0].getId()
cur = self.conn.cursor()
table = "mo_data_" + timestamp_to_str(start, fmt='%Y%m%d')
sql = "insert into {table} values(%s,%s,%s,%s)".format(table=table)
list = []
print 'mo location:', moid, ' size:', len(datas)
for ao in datas:
list.append(ao.dict())
lastest = list[-1] # todo. 还需要检查轨迹点是否有效
try:
cur.execute(sql, [moid, start, Json(list), Json(lastest)])
except:
traceback.print_exc()
instance.getLogger().warning(traceback.format_exc())
def _onDataReady(self, vehicle_id, data):
"""数据接收完成之后进行下一步消息的投递
"""
data = self._time_twiste(data)
if not data:
return
# return data
data = self.buffer.enqueue(vehicle_id, data)
if data: # fresh data
env = data.toEnvelope() # 发送 DataEnvelope
name = self.cfgs.get('post_mq')
serial = env.marshall()
print serial
tick = timestamp_to_str(env.payloads.get(
DataCategory.LOC.value).time,
fmt='%Y%m%d_%H%M%S')
print env.getId(), tick
self.count += 1
print 'data sent:', self.count
mq = AmqpManager.instance().getMessageQueue(name)
mq.produce(serial)