def get_refresh_result_by_sessinId(session_id): """ 根据session_id,返回要查询的信息 Parameters ---------- session_id refresh_result collection 中的session_id Returns 返回refresh_result中的信息 ------- """ # session_id = '5b3c835637d015a9cd07d69e' logger.debug('action sessid_id type is {0}'.format(type(session_id))) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(session_id), num_str) except Exception: logger.debug( 'get refresh_result get number of refresh_result error:%s' % traceback.format_exc()) logger.debug('-------------session_id:{0},str_num:{1}'.format( session_id, str_num)) logger.debug('sessid_id type is {0}'.format(type(session_id))) res = monitor_db['refresh_result' + str_num].find( {'session_id': str(session_id)}) # for x in res: # logger.debug('-------------session_id:{0}'.format(x)) if not res: return [] else: # return assembel_refresh_result(res) return res
def get_all_report_hpcc_dev(_id): """ according _id session_id in refresh_result Args: _id: the session_id in refresh_result Returns:the list of device of all hpcc device """ str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(_id), num_str) except Exception: logger.debug( 'get refresh_result get number of refresh_result error:%s' % traceback.format_exc()) result_r = [] try: result = db_s1['refresh_result' + str_num].find({"session_id": _id}) if not result: return [] else: for res in result: result_r.append(res) return result_r except Exception: logger.debug("get_all_report_hpcc_dev error:%s" % traceback.format_exc()) return result_r
def merge_refresh_new(self, task): session_id = task['session_id'] logger.debug("refresh_router :{}|{}|{}".format(session_id, task.get('host'), task.get('hostname'))) db_key = get_mongo_str(session_id, 10) # if not db_key: # logger.warn('dont get the key:%s' % (session_id)) self.merged_refresh.setdefault(db_key, []).append(task) if len(self.merged_refresh.get(db_key)) > self.package_size: self.update_refresh_result(self.merged_refresh.pop(db_key))
def update_refresh_result(result, flag=1): """ according result, update refresh_result collection of mongo :param result: the content need to be update :param flag: if flag == 1 url update, else dir update :return: json {'msg':xxxx, 'content': xxxx} """ # try: # result_mongo = q_db.refresh_result.find_one({'session_id': result['session_id'], 'name': result['name']}) # logger.debug("find_one refresh_result, session_id:%s, name:%s" % (result['session_id'], result['name'])) # except Exception, e: # logger.debug("receiver_branch update_refresh_result get data from mongo error:%s" % e) # result_mongo = None # if not result_mongo: # try: # db.refresh_result.insert_one(result) # except Exception, e: # logger.debug("receiver_branch update_refresh_result insert_one error:%s" % e) # # return json.dumps( # # {'msg': 'error', 'content': "receiver_branch update_refresh_result insert_one error:%s" % e}) # else: # mongo_id = result_mongo.get("_id", 0) # mongo_result = result_mongo.get('result', 0) # if flag == 1: # mongo_gzip = result_mongo.get('result_gzip', 0) # if mongo_result == 200 or result['result'] == 200: # result['result'] = 200 # if flag == 1: # if mongo_gzip == 200 or result['result_gzip'] == 200: # result['result_gzip'] = 200 # try: # db.refresh_result.update_one({"_id": ObjectId(mongo_id)}, {"$set": result}) # except Exception, e: # logger.debug("receiver_branch update_refresh_result update_one error:%s" % e) logger.debug('update_refresh_result result:%s, flag:%s' % (result, flag)) url_id = result.get('session_id') host = result.get('host') result_code = result.get('result') if flag == 1: result_gzip_code = result.get('result_gzip', '0') if str(result_code) == '200' and str(result_gzip_code) == '200': delete_urlid_host(url_id, host) else: if str(result_code) == '200': delete_urlid_host(url_id, host) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(result.get('session_id')), num_str) except Exception, e: logger.debug('get number of refresh_result error:%s' % traceback.format_exc(e))
def merge_refresh_new(self, task): session_id = task['session_id'] logger.debug("refresh_router :{}|{}|{}".format(session_id, task.get('host'), task.get('hostname'))) db_key = get_mongo_str(session_id, 10) self.merged_refresh.setdefault(db_key, []).append(task) #if len(self.merged_refresh.get(db_key)) > self.package_size: if len(self.merged_refresh.get(db_key)) > 60: logger.debug('pop data is {}'.format( self.merged_refresh.get(db_key))) self.update_refresh_result(self.merged_refresh.pop(db_key))
def update_refresh_result(self, results): logger.debug('update_refresh_result is {}'.format(results)) if not results: return logger.debug('update_refresh_result number:%s' % (len(results))) for result in results: timestr = result['time'] result['time'] = datetime.strptime( timestr, '%Y-%m-%d %H:%M:%S %f' ) # time.mktime(time.strptime(timestr, "%Y-%m-%d %H:%M:%S") # # url_id = result.get('session_id') # # host = result.get('host') # # result_code = result.get('result') # # try: # # #if result.has_key('result_gzip'): # # if 'result_gzip' in result: # # # # result_gzip_code = result.get('result_gzip', '0') # # if str(result_code) == '200' and str(result_gzip_code) == '200': # # delete_urlid_host(url_id, host) # # else: # # if str(result_code) == '200': # # delete_urlid_host(url_id, host) # # except Exception as e: # # #logger.debug('refresh result error :%s' % traceback.format_exc(e)) # # logger.debug('refresh result error :%s' % (e)) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(results[0].get('session_id')), num_str) except Exception as e: logger.debug('get number of refresh_result error:%s' % (e)) try: # db_s1['refresh_result' + str_num].insert_many(results,ordered=False) logger.debug('insert mongodb is {}'.format(results)) db_s1['refresh_result' + str_num].insert_many(results, ordered=False) logger.debug('insert mongodb end') time.sleep(0.05) #logger.debug('update_refresh_result success, session_id:%s' % (result.get('session_id'))) except Exception as e: #logger.warn('update_refresh_result error:%s, session_id:%s' % (traceback.format_exc(e), result.get('session_id'))) # logger.info('update_refresh_result error:%s, session_id:%s' % (e,result.get('session_id'))) logger.warning('insert mongodb is error{}{}'.format( traceback.format_exc(e), results))
def update_refresh_result2(self, results): logger.info('update_refresh_result is {}'.format(results)) if not results: return logger.info('update_refresh_result number:%s' % (len(results))) for result in results: timestr = result['time'] result['time'] = datetime.strptime(timestr,'%Y-%m-%d %H:%M:%S %f') #str_num = '' num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(results[0].get('session_id')), num_str) logger.info('insert mongodb is {}'.format(results)) db_s1['refresh_result' + str_num].insert_many(results, ordered=False) logger.info('insert mongodb end')
def get_all_report_hpcc_dev(_id): """ according _id session_id in refresh_result Args: _id: the session_id in refresh_result Returns:the list of device of all hpcc device """ str_num = '' try: num_str =config.get('refresh_result', 'num') str_num = get_mongo_str(str(_id), num_str) except Exception, e: logger.debug('get refresh_result get number of refresh_result error:%s' % traceback.format_exc(e))
def update_refresh_result(self, results): if not results: return logger.debug('update_refresh_result number:%s' % (len(results))) for result in results: timestr = result['time'] result['time'] = datetime.strptime( timestr, '%Y-%m-%d %H:%M:%S %f' ) #time.mktime(time.strptime(timestr, "%Y-%m-%d %H:%M:%S") url_id = result.get('session_id') host = result.get('host') result_code = result.get('result') try: if 'result_gzip' in result: result_gzip_code = result.get('result_gzip', '0') if str(result_code) == '200' and str( result_gzip_code) == '200': delete_urlid_host(url_id, host) else: if str(result_code) == '200': delete_urlid_host(url_id, host) except Exception: logger.debug('refresh result error :%s' % traceback.format_exc()) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(results[0].get('session_id')), num_str) except Exception: logger.debug('get number of refresh_result error:%s' % traceback.format_exc()) try: db_s1['refresh_result' + str_num].insert_many(results, ordered=False) time.sleep(0.05) logger.debug('update_refresh_result success, session_id:%s' % (result.get('session_id'))) except Exception: logger.info('update_refresh_result error:%s, session_id:%s' % (traceback.format_exc(), result.get('session_id')))
def insert_refresh_result(self, urls, devices): """ according urls and devices, insert refresh_result Args: urls: devices: Returns: """ devices_copy = copy.deepcopy(devices) hpcc_dev = [] try: if urls: for url in urls: _id = str(url.get('id')) for dev in devices_copy: dev['session_id'] = _id dev['result'] = '0' dev['result_gzip'] = '0' dev['time_insert'] = datetime.now() if dev.get('type') == 'HPCC' and dev.get( 'status') == 'OPEN': hpcc_dev.append(dev) logger.debug('refresh session_id:%s' % _id) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = tools.get_mongo_str(str(_id), num_str) except Exception: logger.debug('get number of refresh_result error:%s' % traceback.format_exc()) db_s1['refresh_result' + str_num].insert(hpcc_dev) hpcc_dev = [] for dev in devices_copy: if '_id' in dev: dev.pop("_id") logger.debug( 'insert_refresh_result insert refresh_result success data') except Exception: logger.info('insert_refresh_result error:%s' % traceback.format_exc()) return
def get_refresh_result_by_sessinId(session_id): """ 根据session_id,返回要查询的信息 Parameters ---------- session_id refresh_result collection 中的session_id Returns 返回refresh_result中的信息 ------- """ # session_id = '5b3c835637d015a9cd07d69e' logger.debug('action sessid_id type is {0}'.format(type(session_id))) str_num = '' try: num_str = config.get('refresh_result', 'num') str_num = get_mongo_str(str(session_id), num_str) except Exception, e: logger.debug( 'get refresh_result get number of refresh_result error:%s' % traceback.format_exc(e))
def merge_refresh_new(self, task): session_id = task['session_id'] vip = task['host'] host =vip # vip.replace('.','#') result = task['result'] logger.info("refresh_router :{}|{}|{}".format(session_id, task.get('host'), task.get('hostname'))) db_key = get_mongo_str(session_id, 10) #logger.info("refresh_router_key :{}|{}".format(session_id,db_key)) insert_dict = {host:result} #添加过滤条件 if not REDIS_CACHE.hexists(session_id,host): REDIS_CACHE.hmset(session_id,insert_dict) REDIS_CACHE.expire(session_id,60*60*24) self.merged_refresh.setdefault(db_key, []).append(task) #if len(self.merged_refresh.get(db_key)) > self.package_size: if len(self.merged_refresh.get(db_key)) > 60: logger.info('pop data is {}'.format(self.merged_refresh.get(db_key))) self.update_refresh_result(self.merged_refresh.pop(db_key)) else: REDIS_CACHE.hmset(session_id, insert_dict) REDIS_CACHE.expire(session_id, 60 * 24)
result_code = result.get('result') try: if result.has_key('result_gzip'): result_gzip_code = result.get('result_gzip', '0') if str(result_code) == '200' and str(result_gzip_code) == '200': delete_urlid_host(url_id, host) else: if str(result_code) == '200': delete_urlid_host(url_id, host) except Exception,e: logger.debug('refresh result error :%s' % traceback.format_exc(e)) str_num = '' try: num_str =config.get('refresh_result', 'num') str_num = get_mongo_str(str(results[0].get('session_id')), num_str) except Exception, e: logger.debug('get number of refresh_result error:%s' % traceback.format_exc(e)) try: db_s1['refresh_result' + str_num].insert_many(results,ordered=False) time.sleep(0.05) logger.debug('update_refresh_result success, session_id:%s' % (result.get('session_id'))) except Exception, e: logger.info('update_refresh_result error:%s, session_id:%s' % (traceback.format_exc(e), result.get('session_id'))) def load_task(self,task): '''