class TestYAML(object):
def __init__(self):
self.my_ini = MyYAML()
def get_ini(self):
print self.my_ini.get_ini()
print list(self.my_ini.get_ini()['mobiles'])
def set_ini(self):
data = self.my_ini.get_ini()
print 'data=' % data
data['mobiles'] = ['123', '678']
print self.my_ini.set_ini(data)
class TestYAML(object):
def __init__(self):
self.my_ini = MyYAML()
def get_ini(self):
print dict(self.my_ini.get_ini()['kakou'])
print dict(self.my_ini.get_ini()['union'])
print list(self.my_ini.get_ini()['usefulkkdd'])
print type(self.my_ini.get_ini()['city'])
print type(self.my_ini.get_ini()['kkdd_id'])
print type(self.my_ini.get_ini()['id_flag'])
print self.my_ini.get_ini()['id_step']
def set_ini(self):
data = self.my_ini.get_ini()
data['usefulkkdd'] = ['789', '456', '123', 'user']
print self.my_ini.set_ini(data)
class UploadData(object):
def __init__(self):
# 配置文件
self.ini = MyYAML('/home/my.yaml')
self.flag_ini = MyYAML('/home/flag.yaml')
self.my_ini = self.ini.get_ini()
# request方法类
self.kk = Kakou(**dict(self.my_ini['kakou']))
self.uk = UnionKakou(**dict(self.my_ini['union']))
self.sq = KakouDB('/home/kakou.db')
self.kk.status = True
self.uk.status = True
self.city = self.my_ini['city']
self.kkdd_id = self.my_ini['kkdd_id']
self.id_flag = self.flag_ini.get_ini()['id']
self.step = self.my_ini['id_step']
# 有效的卡口地点
if self.my_ini['usefulkkdd'] is None:
self.useful_kkdd = set()
else:
self.useful_kkdd = set(self.my_ini['usefulkkdd'])
def set_id(self, _id, msg=''):
"""设置ID"""
self.id_flag = _id
self.flag_ini.set_ini({'id': _id})
logger.info('{0} {1}'.format(_id, msg))
def post_data(self, start_id, end_id):
"""上传卡口数据"""
info = self.kk.get_kakou(start_id, end_id, 1, self.step+1)
# 如果查询数据为0则退出
if info['total_count'] == 0:
return
data = []
for i in info['items']:
if i['kkbh'] is None:
i['kkdd_id'] = self.kkdd_id
i['kkbh'] = self.kkdd_id
elif len(i['kkbh']) != 9:
i['kkdd_id'] = self.kkdd_id
i['kkbh'] = self.kkdd_id
# 有效卡点为零时
if len(self.useful_kkdd) == 0:
pass
elif i['kkdd_id'] not in self.useful_kkdd:
continue
data.append({'jgsj': i['jgsj'], # 经过时间
'hphm': i['hphm'], # 号牌号码
'kkdd_id': i['kkbh'], # 卡口地点ID
'hpys_id': i['hpys_id'], # 号牌颜色ID
'fxbh': i['fxbh_code'], # 方向编号
'cdbh': i['cdbh'], # 车道
'clsd': i['clsd'], # 车速
'hpzl': i['hpzl'], # 号牌种类
'img_path': i['imgurl']}) # 图片url地址
if len(data) > 0:
self.uk.post_kakou(data) # 上传数据
def post_info_realtime(self):
print('id_flag: {0}'.format(self.id_flag))
"""上传实时数据"""
maxid = self.kk.get_maxid()
# id间隔
interval = maxid - self.id_flag
#print('interval={0}'.format(interval))
# 没有新数据则返回
if interval <= 0:
r = self.post_data_from_db()
return r
# id间隔大于阀值
if interval > self.step * 60:
for i in range(60):
self.sq.add_idflag(self.id_flag+1, self.id_flag+self.step)
self.set_id(self.id_flag+self.step, msg='sqlite') # 设置最新ID
return 0
# id间隔小于步长
if interval < self.step:
self.post_data(self.id_flag+1, maxid)
self.set_id(maxid) # 设置最新ID
return 0.5
self.post_data(self.id_flag+1, self.id_flag+self.step)
self.set_id(self.id_flag+self.step)
return 0.25
def post_data_from_db(self):
"""上传历史数据"""
r = self.sq.get_idflag(banned=0, limit=1)
if r == []:
return 1
self.post_data(r[0][1], r[0][2])
# 删除历史ID
self.sq.del_idflag(r[0][0])
logger.info('{0} {1}'.format(r[0][2], 'from db'))
return 0
def main_loop(self):
while 1:
if self.kk.status and self.uk.status:
try:
n = self.post_info_realtime()
time.sleep(n)
except Exception as e:
logger.exception(e)
time.sleep(5)
else:
try:
print(self.kk.status)
print(self.uk.status)
if not self.kk.status:
self.kk.get_maxid()
self.kk.status = True
if not self.uk.status:
self.uk.connect_test()
self.uk.status = True
except Exception as e:
logger.exception(e)
time.sleep(1)
class Dumper(object):
def __init__(self):
self.ini = MyYAML('my.yaml')
self.my_ini = self.ini.get_ini()
self.flag_ini = MyYAML('flag.yaml')
self.last_time = arrow.get(self.flag_ini.get_ini()['last_time'])
self.backup_path = self.my_ini['backup_path']
self.interval = self.my_ini['interval']
self.gc = self.my_ini['gc']
# 创建数据库
db = TinyDB('db.json')
self.table = db.table('dump')
def set_flag(self, date, msg=''):
self.last_time = date
self.flag_ini.set_ini(
{'last_time': date.format('YYYY-MM-DDTHH:mm:ssZZ')})
logger.info('{0} {1}'.format(date.format('YYYY-MM-DDTHH:mm:ssZZ'),
msg))
def dump(self, ini, date):
# 保存位置
folder = '{0}/{1}/{2}'.format(self.backup_path, ini['db'],
date.format('YYYYMMDDTHHmmss'))
if not os.path.isdir('{0}/{1}'.format(self.backup_path, ini['db'])):
os.makedirs('{0}/{1}'.format(self.backup_path, ini['db']))
# shell命令
cmd = '/root/tidb-enterprise-tools-latest-linux-amd64/bin/mydumper -h {0} -P {1} -u {2} -p {3} -t 16 -F 64 -B {4} --skip-tz-utc -o {5}'.format(
ini['host'], ini['port'], ini['user'], ini['pwd'], ini['db'],
folder)
child = subprocess.Popen(cmd, shell=True)
child.wait()
self.table.insert({
'cmd': cmd,
'folder': folder,
'created_date': date.format('YYYY-MM-DDTHH:mm:ssZZ')
})
print(cmd)
logger.info(cmd)
def clean(self, gc):
dumper = Query()
items = self.table.all()
if items == []:
return
created_date = arrow.get(items[0]['created_date'])
if (time.time() - created_date.timestamp) > gc * 24 * 60 * 60.0:
cmd = 'rm -rf {0}'.format(items[0]['folder'])
child = subprocess.Popen(cmd, shell=True)
child.wait()
self.table.remove(doc_ids=[items[0].doc_id])
logger.info('{0} has been removed from TinyDB'.format(items[0]))
def time_check(self, now):
if (now.timestamp -
self.last_time.timestamp) < self.interval * 60 * 60.0:
return False
return True
def run(self):
while 1:
try:
now = arrow.now('PRC')
if self.time_check(now):
self.dump(dict(self.my_ini['mysql']), now)
self.set_flag(now)
self.clean(self.gc)
time.sleep(5)
except Exception as e:
logger.exception(e)
time.sleep(30)
class UploadData(object):
def __init__(self):
# 配置文件
self.my_ini = MyYAML('my.yaml').get_ini()
self.flag_ini = MyYAML('flag.yaml')
# request方法类
self.kk = Kakou(**dict(self.my_ini['kakou']))
self.tk = TempKakou(**dict(self.my_ini['temp']))
self.kk.status = True
self.tk.status = True
# ID上传标记
self.id_flag = self.flag_ini.get_ini()['id']
self.step = self.my_ini['step']
def set_id(self, _id):
"""设置ID"""
self.id_flag = _id
self.flag_ini.set_ini({"id": _id})
print(_id)
def post_info(self):
"""上传数据"""
car_info = self.kk.get_kakou(self.id_flag + 1,
self.id_flag + self.step)
# 如果查询数据为0
if car_info['total_count'] == 0:
time.sleep(1)
return
data = []
for i in car_info['items']:
if i['kkdd_id'] is None or i['kkdd_id'] == '':
i['kkdd_id'] = '441322000'
if i['hphm'] is None or i['hphm'] == '':
i['hphm'] = '-'
if i['id'] > 0:
i['imgpath'] = ''
data.append(i)
i['imgurl'] = i['imgurl'].replace("10.44.245.247:8083",
"10.47.223.151:8099/blkk")
r = self.tk.post_final(data) #上传数据
# 设置最新ID
self.set_id(car_info['items'][-1]['id'])
def main_loop(self):
while 1:
if self.kk.status and self.tk.status:
#print('test')
try:
self.post_info()
time.sleep(0.5)
except Exception as e:
print(e)
time.sleep(1)
else:
try:
if not self.kk.status:
self.kk.get_kakou(214542651, 214542653)
self.kk.status = True
if not self.tk.status:
self.tk.connect_test()
self.tk.status = True
except Exception as e:
time.sleep(1)
class UploadData(object):
def __init__(self):
# 配置文件
self.ini = MyYAML('/home/my.yaml')
self.my_ini = self.ini.get_ini()
self.flag_ini = MyYAML('/home/flag.yaml')
# request方法类
self.hd = Kakou(**dict(self.my_ini['hd']))
self.ys = Kakou(**dict(self.my_ini['ys']))
self.con = ConsulAPI()
self.hd.status = True
self.ys.status = True
self.step = self.my_ini['step']
self.uuid = None # session id
self.session_time = time.time() # session生成时间戳
self.ttl = dict(self.my_ini['consul'])['ttl'] # 生存周期
self.lock_name = dict(self.my_ini['consul'])['lock_name'] # 锁名
self.local_ip = '10.47.223.148' #socket.gethostbyname(socket.gethostname()) # 本地IP
self.maxid = 0
self.id_flag = self.flag_ini.get_ini()['id']
def get_id2(self):
"""获取上传id"""
r = self.con.get_id()[0]
return json.loads(base64.b64decode(
r['Value']).decode()), r['ModifyIndex']
def get_id(self):
"""获取上传id"""
return self.id_flag
def set_id2(self, _id, modify_index):
"""设置ID"""
if self.con.put_id(_id, modify_index):
print(_id)
def set_id(self, _id, msg=''):
"""设置ID"""
self.id_flag = _id
self.flag_ini.set_ini({'id': _id})
print(self.id_flag)
logger.info('{0} {1}'.format(_id, msg))
def get_lost(self):
"""获取未上传数据id列表"""
r = self.con.get_lost()[0]
return json.loads(base64.b64decode(
r['Value']).decode()), r['ModifyIndex']
def get_lock(self):
"""获取锁"""
if self.uuid is None:
self.uuid = self.con.put_session(self.ttl, self.lock_name)['ID']
self.session_time = time.time()
p = False
# 大于一定时间间隔则更新session
# t = time.time() - self.session_time
if (time.time() - self.session_time) > (self.ttl - 10):
self.con.renew_session(self.uuid)
self.session_time = time.time()
p = True
l = self.con.get_lock(self.uuid, self.local_ip)
if p:
print(self.uuid, l)
# session过期
if l == None:
self.uuid = None
return False
return l
def post_info(self):
"""上传数据"""
#id, modify_index = self.get_id()
id = self.get_id()
if self.maxid == 0 or self.maxid <= id:
self.maxid = self.ys.get_maxid()
return 0
if self.maxid <= (id + self.step):
last_id = self.maxid
else:
last_id = id + self.step
info = self.ys.get_kakou(id + 1, last_id, 1, self.step + 1)
#print('maxid=%s'%self.maxid)
#print('total_count=%s'%info['total_count'])
# 如果查询数据为0
if info['total_count'] == 0:
self.set_id(last_id, modify_index)
return 0
items = []
for i in info['items']:
i['tjtp'] = helper.created_url(i['imgurl'])
items.append(i)
if len(items) > 0:
self.hd.post_kakou(items)
# 设置最新ID
#self.set_id(last_id, modify_index)
self.set_id(last_id)
return info['total_count']
def main_loop(self):
while 1:
try:
#if not self.get_lock():
# time.sleep(2)
# continue
n = self.post_info()
if n < self.step:
time.sleep(0.5)
except Exception as e:
logger.exception(e)
time.sleep(15)
class UploadData(object):
def __init__(self):
# 配置文件
self.ini = MyYAML('/home/my.yaml')
self.flag_ini = MyYAML('/home/flag.yaml')
self.my_ini = self.ini.get_ini()
# request方法类
self.kk = Kakou(**dict(self.my_ini['kakou']))
self.uk = UnionKakou(**dict(self.my_ini['union']))
self.sq = KakouDB('/home/kakou.db')
self.kk.status = True
self.uk.status = True
self.city = self.my_ini['city']
self.kkdd_id = self.my_ini['kkdd_id']
self.id_flag = self.flag_ini.get_ini()['id']
self.step = self.my_ini['id_step']
# 有效的卡口地点
if self.my_ini['usefulkkdd'] is None:
self.useful_kkdd = set()
else:
self.useful_kkdd = set(self.my_ini['usefulkkdd'])
def set_id(self, _id, msg=''):
"""设置ID"""
self.id_flag = _id
self.flag_ini.set_ini({'id': _id})
logger.info('{0} {1}'.format(_id, msg))
def post_data(self, start_id, end_id):
"""上传卡口数据"""
info = self.kk.get_kakou(start_id, end_id, 1, self.step + 1)
# 如果查询数据为0则退出
if info['total_count'] == 0:
return
data = []
for i in info['items']:
if i['kkbh'] is None:
i['kkdd_id'] = self.kkdd_id
i['kkbh'] = self.kkdd_id
elif len(i['kkbh']) != 9:
i['kkdd_id'] = self.kkdd_id
i['kkbh'] = self.kkdd_id
#if i['kkbh'] is None:
# continue
#if len(i['kkbh']) != 9:
# continue
# 有效卡点为零时
if len(self.useful_kkdd) == 0:
pass
elif i['kkbh'] not in self.useful_kkdd:
continue
data.append({
'jgsj': i['jgsj'], # 经过时间
'hphm': i['hphm'], # 号牌号码
'kkdd_id': i['kkbh'], # 卡口地点ID
'hpys_id': i['hpys_id'], # 号牌颜色ID
'fxbh': i['fxbh_code'], # 方向编号
'cdbh': i['cdbh'], # 车道
'clsd': i['clsd'], # 车速
'hpzl': i['hpzl'], # 号牌种类
'img_path': i['imgurl']
}) # 图片url地址
if len(data) > 0:
self.uk.post_kakou(data) # 上传数据
def post_info_realtime(self):
print('id_flag: {0}'.format(self.id_flag))
"""上传实时数据"""
maxid = self.kk.get_maxid()
# id间隔
interval = maxid - self.id_flag
#print('interval={0}'.format(interval))
# 没有新数据则返回
if interval <= 0:
r = self.post_data_from_db()
return r
# id间隔大于阀值
if interval > self.step * 60:
for i in range(60):
self.sq.add_idflag(self.id_flag + 1, self.id_flag + self.step)
self.set_id(self.id_flag + self.step, msg='sqlite') # 设置最新ID
return 0
# id间隔小于步长
if interval < self.step:
self.post_data(self.id_flag + 1, maxid)
self.set_id(maxid) # 设置最新ID
return 0.5
self.post_data(self.id_flag + 1, self.id_flag + self.step)
self.set_id(self.id_flag + self.step)
return 0.25
def post_data_from_db(self):
"""上传历史数据"""
r = self.sq.get_idflag(banned=0, limit=1)
if r == []:
return 1
self.post_data(r[0][1], r[0][2])
# 删除历史ID
self.sq.del_idflag(r[0][0])
logger.info('{0} {1}'.format(r[0][2], 'from db'))
return 0
def main_loop(self):
while 1:
if self.kk.status and self.uk.status:
try:
n = self.post_info_realtime()
time.sleep(n)
except Exception as e:
logger.exception(e)
time.sleep(5)
else:
try:
print(self.kk.status)
print(self.uk.status)
if not self.kk.status:
self.kk.get_maxid()
self.kk.status = True
if not self.uk.status:
self.uk.get_root()
self.uk.status = True
except Exception as e:
logger.exception(e)
time.sleep(1)