def get_refresh_result_by_sessinId(session_id):
"""
根据session_id，返回要查询的信息
Parameters
----------
session_id refresh_result collection 中的session_id
Returns 返回refresh_result中的信息
-------
"""
# session_id = '5b3c835637d015a9cd07d69e'
logger.debug('action sessid_id type is {0}'.format(type(session_id)))
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(session_id), num_str)
except Exception:
logger.debug(
'get refresh_result get number of refresh_result error:%s' %
traceback.format_exc())
logger.debug('-------------session_id:{0},str_num:{1}'.format(
session_id, str_num))
logger.debug('sessid_id type is {0}'.format(type(session_id)))
res = monitor_db['refresh_result' + str_num].find(
{'session_id': str(session_id)})
# for x in res:
# logger.debug('-------------session_id:{0}'.format(x))
if not res:
return []
else:
# return assembel_refresh_result(res)
return res
def get_all_report_hpcc_dev(_id):
"""
according _id session_id in refresh_result
Args:
_id: the session_id in refresh_result
Returns:the list of device of all hpcc device
"""
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(_id), num_str)
except Exception:
logger.debug(
'get refresh_result get number of refresh_result error:%s' %
traceback.format_exc())
result_r = []
try:
result = db_s1['refresh_result' + str_num].find({"session_id": _id})
if not result:
return []
else:
for res in result:
result_r.append(res)
return result_r
except Exception:
logger.debug("get_all_report_hpcc_dev error:%s" %
traceback.format_exc())
return result_r
def merge_refresh_new(self, task):
session_id = task['session_id']
logger.debug("refresh_router :{}|{}|{}".format(session_id,
task.get('host'),
task.get('hostname')))
db_key = get_mongo_str(session_id, 10)
# if not db_key:
# logger.warn('dont get the key:%s' % (session_id))
self.merged_refresh.setdefault(db_key, []).append(task)
if len(self.merged_refresh.get(db_key)) > self.package_size:
self.update_refresh_result(self.merged_refresh.pop(db_key))
def update_refresh_result(result, flag=1):
"""
according result, update refresh_result collection of mongo
:param result: the content need to be update
:param flag: if flag == 1 url update, else dir update
:return: json {'msg':xxxx, 'content': xxxx}
"""
# try:
# result_mongo = q_db.refresh_result.find_one({'session_id': result['session_id'], 'name': result['name']})
# logger.debug("find_one refresh_result, session_id:%s, name:%s" % (result['session_id'], result['name']))
# except Exception, e:
# logger.debug("receiver_branch update_refresh_result get data from mongo error:%s" % e)
# result_mongo = None
# if not result_mongo:
# try:
# db.refresh_result.insert_one(result)
# except Exception, e:
# logger.debug("receiver_branch update_refresh_result insert_one error:%s" % e)
# # return json.dumps(
# # {'msg': 'error', 'content': "receiver_branch update_refresh_result insert_one error:%s" % e})
# else:
# mongo_id = result_mongo.get("_id", 0)
# mongo_result = result_mongo.get('result', 0)
# if flag == 1:
# mongo_gzip = result_mongo.get('result_gzip', 0)
# if mongo_result == 200 or result['result'] == 200:
# result['result'] = 200
# if flag == 1:
# if mongo_gzip == 200 or result['result_gzip'] == 200:
# result['result_gzip'] = 200
# try:
# db.refresh_result.update_one({"_id": ObjectId(mongo_id)}, {"$set": result})
# except Exception, e:
# logger.debug("receiver_branch update_refresh_result update_one error:%s" % e)
logger.debug('update_refresh_result result:%s, flag:%s' % (result, flag))
url_id = result.get('session_id')
host = result.get('host')
result_code = result.get('result')
if flag == 1:
result_gzip_code = result.get('result_gzip', '0')
if str(result_code) == '200' and str(result_gzip_code) == '200':
delete_urlid_host(url_id, host)
else:
if str(result_code) == '200':
delete_urlid_host(url_id, host)
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(result.get('session_id')), num_str)
except Exception, e:
logger.debug('get number of refresh_result error:%s' %
traceback.format_exc(e))
def merge_refresh_new(self, task):
session_id = task['session_id']
logger.debug("refresh_router :{}|{}|{}".format(session_id,
task.get('host'),
task.get('hostname')))
db_key = get_mongo_str(session_id, 10)
self.merged_refresh.setdefault(db_key, []).append(task)
#if len(self.merged_refresh.get(db_key)) > self.package_size:
if len(self.merged_refresh.get(db_key)) > 60:
logger.debug('pop data is {}'.format(
self.merged_refresh.get(db_key)))
self.update_refresh_result(self.merged_refresh.pop(db_key))
def update_refresh_result(self, results):
logger.debug('update_refresh_result is {}'.format(results))
if not results:
return
logger.debug('update_refresh_result number:%s' % (len(results)))
for result in results:
timestr = result['time']
result['time'] = datetime.strptime(
timestr, '%Y-%m-%d %H:%M:%S %f'
) # time.mktime(time.strptime(timestr, "%Y-%m-%d %H:%M:%S")
# # url_id = result.get('session_id')
# # host = result.get('host')
# # result_code = result.get('result')
# # try:
# # #if result.has_key('result_gzip'):
# # if 'result_gzip' in result:
# #
# # result_gzip_code = result.get('result_gzip', '0')
# # if str(result_code) == '200' and str(result_gzip_code) == '200':
# # delete_urlid_host(url_id, host)
# # else:
# # if str(result_code) == '200':
# # delete_urlid_host(url_id, host)
# # except Exception as e:
# # #logger.debug('refresh result error :%s' % traceback.format_exc(e))
# # logger.debug('refresh result error :%s' % (e))
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(results[0].get('session_id')), num_str)
except Exception as e:
logger.debug('get number of refresh_result error:%s' % (e))
try:
# db_s1['refresh_result' + str_num].insert_many(results,ordered=False)
logger.debug('insert mongodb is {}'.format(results))
db_s1['refresh_result' + str_num].insert_many(results,
ordered=False)
logger.debug('insert mongodb end')
time.sleep(0.05)
#logger.debug('update_refresh_result success, session_id:%s' % (result.get('session_id')))
except Exception as e:
#logger.warn('update_refresh_result error:%s, session_id:%s' % (traceback.format_exc(e), result.get('session_id')))
# logger.info('update_refresh_result error:%s, session_id:%s' % (e,result.get('session_id')))
logger.warning('insert mongodb is error{}{}'.format(
traceback.format_exc(e), results))
def update_refresh_result2(self, results):
logger.info('update_refresh_result is {}'.format(results))
if not results:
return
logger.info('update_refresh_result number:%s' % (len(results)))
for result in results:
timestr = result['time']
result['time'] = datetime.strptime(timestr,'%Y-%m-%d %H:%M:%S %f')
#str_num = ''
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(results[0].get('session_id')), num_str)
logger.info('insert mongodb is {}'.format(results))
db_s1['refresh_result' + str_num].insert_many(results, ordered=False)
logger.info('insert mongodb end')
def get_all_report_hpcc_dev(_id):
"""
according _id session_id in refresh_result
Args:
_id: the session_id in refresh_result
Returns:the list of device of all hpcc device
"""
str_num = ''
try:
num_str =config.get('refresh_result', 'num')
str_num = get_mongo_str(str(_id), num_str)
except Exception, e:
logger.debug('get refresh_result get number of refresh_result error:%s' % traceback.format_exc(e))
def update_refresh_result(self, results):
if not results:
return
logger.debug('update_refresh_result number:%s' % (len(results)))
for result in results:
timestr = result['time']
result['time'] = datetime.strptime(
timestr, '%Y-%m-%d %H:%M:%S %f'
) #time.mktime(time.strptime(timestr, "%Y-%m-%d %H:%M:%S")
url_id = result.get('session_id')
host = result.get('host')
result_code = result.get('result')
try:
if 'result_gzip' in result:
result_gzip_code = result.get('result_gzip', '0')
if str(result_code) == '200' and str(
result_gzip_code) == '200':
delete_urlid_host(url_id, host)
else:
if str(result_code) == '200':
delete_urlid_host(url_id, host)
except Exception:
logger.debug('refresh result error :%s' %
traceback.format_exc())
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(results[0].get('session_id')), num_str)
except Exception:
logger.debug('get number of refresh_result error:%s' %
traceback.format_exc())
try:
db_s1['refresh_result' + str_num].insert_many(results,
ordered=False)
time.sleep(0.05)
logger.debug('update_refresh_result success, session_id:%s' %
(result.get('session_id')))
except Exception:
logger.info('update_refresh_result error:%s, session_id:%s' %
(traceback.format_exc(), result.get('session_id')))
def insert_refresh_result(self, urls, devices):
"""
according urls and devices, insert refresh_result
Args:
urls:
devices:
Returns:
"""
devices_copy = copy.deepcopy(devices)
hpcc_dev = []
try:
if urls:
for url in urls:
_id = str(url.get('id'))
for dev in devices_copy:
dev['session_id'] = _id
dev['result'] = '0'
dev['result_gzip'] = '0'
dev['time_insert'] = datetime.now()
if dev.get('type') == 'HPCC' and dev.get(
'status') == 'OPEN':
hpcc_dev.append(dev)
logger.debug('refresh session_id:%s' % _id)
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = tools.get_mongo_str(str(_id), num_str)
except Exception:
logger.debug('get number of refresh_result error:%s' %
traceback.format_exc())
db_s1['refresh_result' + str_num].insert(hpcc_dev)
hpcc_dev = []
for dev in devices_copy:
if '_id' in dev:
dev.pop("_id")
logger.debug(
'insert_refresh_result insert refresh_result success data')
except Exception:
logger.info('insert_refresh_result error:%s' %
traceback.format_exc())
return
def get_refresh_result_by_sessinId(session_id):
"""
根据session_id，返回要查询的信息
Parameters
----------
session_id refresh_result collection 中的session_id
Returns 返回refresh_result中的信息
-------
"""
# session_id = '5b3c835637d015a9cd07d69e'
logger.debug('action sessid_id type is {0}'.format(type(session_id)))
str_num = ''
try:
num_str = config.get('refresh_result', 'num')
str_num = get_mongo_str(str(session_id), num_str)
except Exception, e:
logger.debug(
'get refresh_result get number of refresh_result error:%s' %
traceback.format_exc(e))
def merge_refresh_new(self, task):
session_id = task['session_id']
vip = task['host']
host =vip # vip.replace('.','#')
result = task['result']
logger.info("refresh_router :{}|{}|{}".format(session_id, task.get('host'), task.get('hostname')))
db_key = get_mongo_str(session_id, 10)
#logger.info("refresh_router_key :{}|{}".format(session_id,db_key))
insert_dict = {host:result}
#添加过滤条件
if not REDIS_CACHE.hexists(session_id,host):
REDIS_CACHE.hmset(session_id,insert_dict)
REDIS_CACHE.expire(session_id,60*60*24)
self.merged_refresh.setdefault(db_key, []).append(task)
#if len(self.merged_refresh.get(db_key)) > self.package_size:
if len(self.merged_refresh.get(db_key)) > 60:
logger.info('pop data is {}'.format(self.merged_refresh.get(db_key)))
self.update_refresh_result(self.merged_refresh.pop(db_key))
else:
REDIS_CACHE.hmset(session_id, insert_dict)
REDIS_CACHE.expire(session_id, 60 * 24)
result_code = result.get('result')
try:
if result.has_key('result_gzip'):
result_gzip_code = result.get('result_gzip', '0')
if str(result_code) == '200' and str(result_gzip_code) == '200':
delete_urlid_host(url_id, host)
else:
if str(result_code) == '200':
delete_urlid_host(url_id, host)
except Exception,e:
logger.debug('refresh result error :%s' % traceback.format_exc(e))
str_num = ''
try:
num_str =config.get('refresh_result', 'num')
str_num = get_mongo_str(str(results[0].get('session_id')), num_str)
except Exception, e:
logger.debug('get number of refresh_result error:%s' % traceback.format_exc(e))
try:
db_s1['refresh_result' + str_num].insert_many(results,ordered=False)
time.sleep(0.05)
logger.debug('update_refresh_result success, session_id:%s' % (result.get('session_id')))
except Exception, e:
logger.info('update_refresh_result error:%s, session_id:%s' % (traceback.format_exc(e),
result.get('session_id')))
def load_task(self,task):
'''
