def logMonitor_Tx(msg, params, password, nw): LOGMSG_GRP = nw['LOGMSG_GRP'] LOGMSG_PORT = nw['LOGMSG_PORT'] sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32) print("Starting Tx process...\n") localnode = params['localnode'] role = params['role'] node = params['localnode'] cluster = params['cluster_id'] hostname = (socket.gethostname()) jobs = [] z = multiprocessing.Process(target=nm.infHBeat, args=(params, nw)) jobs.append(z) z.daemon = True z.start() n = multiprocessing.Process(target=nm.messageTagGen, args=(nw, )) jobs.append(n) n.daemon = True n.start() if role == 'RxTx': p = multiprocessing.Process(target=logMonitor_Rx, args=( password, params, )) jobs.append(p) p.daemon = True p.start() ds = multiprocessing.Process(target=sl.deepSearch) jobs.append(ds) ds.daemon = True ds.start() q = multiprocessing.Process(target=syslog) jobs.append(q) q.daemon = True q.start() lenfr = 0 send_throttle = 2 lfr = [0, 0] while True: lfr[0] = lfr[1] if max(lfr) > 100: with open('syslog.log', 'w') as f: f.close() lfr = [0, 0] time.sleep(send_throttle) try: with open('droplist.json', 'r') as dlj: drop_tag = json.load(dlj) drop_tag = str(drop_tag) except: print('possible JSONDecodeError') drop_tag = '[]' pass while True: with open('syslog.log', 'r') as f: fr = f.readlines() lfr[1] = len(fr) if lfr[1] > lfr[0]: msg = '' for i in fr[lfr[0]:lfr[1]]: msg = i.rstrip() parse_msg = json.loads(msg) ts = time.time() msg = { 'source_time': ts, 'sending_node': localnode, 'sending_hostname': hostname, 'orig_message': parse_msg['log_message'], 'orig_addr': parse_msg['orig_addr'], 'drop_tag': drop_tag } msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) try: sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT)) except OSError: msg = ( "Attempting to send %s log messages from overran Tx buffer" % str(len(fr))) msg = localnode + '@' + hostname + "# " + '"' + msg + '"' + drop_tag msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT)) pass if lfr[0] == lfr[1]: pass else: pass break sys.exit()
def searchMem(search_list, dcdmsg, password, search_term_debug='off'): with open('network_cfg.json', 'r') as nwc: nw = json.load(nwc) SCH_GRP = nw['SCH_GRP'] SCH_PORT = nw['SCH_PORT'] sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32) js_dcdmsg = json.loads(dcdmsg) so = js_dcdmsg['search_oper'] sv = js_dcdmsg['search_var'] sf = js_dcdmsg['search_field'] tmn = js_dcdmsg['time_min'] tmx = js_dcdmsg['time_max'] search_result = [] if so == 'OR': search_result = [ j for j in search_list if any(x in j[sf] for x in sv) ] search_result = [ j for j in search_result if j["source_time"] > tmn and j["source_time"] < tmx ] if so == 'AND': search_result = [ j for j in search_list if all(x in j[sf] for x in sv) ] else: print("Using 'OR' operator") search_result = [ j for j in search_list if any(x in j[sf] for x in sv) ] search_result = [ j for j in search_result if j["source_time"] > tmn and j["source_time"] < tmx ] if search_term_debug == 'on': print(so) print('is the search operator') print(sv) print('is search var\n') print(sf) print('is search field\n') print(search_result) print('is search result\n\n') print(len(search_result)) print('is the number of search results###\n\n') '''need to throttle responses using loop to limit msg size''' ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') print(st + ' is START time for sending memory search responses') i = 0 send_chunk = [] try: if search_result is not []: for k in search_result: send_chunk.append(k) i += 1 if i % 40 == 0: js_search_result = json.dumps(send_chunk) js_search_result = bytes(js_search_result, "ascii") msg = cryp.EncryptMsg(js_search_result, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) send_chunk = [] time.sleep(.2) if send_chunk != []: js_search_result = json.dumps(send_chunk) js_search_result = bytes(js_search_result, "ascii") msg = cryp.EncryptMsg(js_search_result, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) else: pass ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') print(st + ' is END time for sending memory search responses') except OSError: msg = ( "Attempting to send %s log messages from overran Tx buffer" % str(len(js_search_result))) msg = localnode + '@' + hostname + "# " + '"' + msg + '"' + drop_tag msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) pass return search_result
def searchSend(input_type='user', search_input=[], search_oper='AND', search_type='Q', api_file=None, time_min=0, time_max=30000000000): with open('pwf', 'r') as p: password = p.read() password = password.rstrip() with open('network_cfg.json', 'r') as nwc: nw = json.load(nwc) LOGMSG_GRP = nw['LOGMSG_GRP'] LOGMSG_PORT = nw['LOGMSG_PORT'] SCH_GRP = nw['SCH_GRP'] SCH_PORT = nw['SCH_PORT'] DEEP_SCH_GRP = nw['DEEP_SCH_GRP'] DEEP_SCH_PORT = nw['DEEP_SCH_PORT'] ''' may want to break out cache search to help with scaling SEARCH_CACHE_GRP='232.1.1.12' SEARCH_CACHE_PORT = 5012 ''' sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32) if input_type == 'user': in_field = '' search_input = [] while '!!!' not in in_field: in_field = str( input("Please enter search string, enter !!! to end: ")) search_input.append(in_field) search_input.pop() print(search_input) search_oper = str( input("Enter your search operator as AND , OR -> ")) search_type = str( input( "Enter search type Q:Quick, D:Deep:, C:Cached or ZM/ZD:Zero-Mem/Disk-Cache,W:Write-Cache -> " )) if search_type == 'Q': search_tag = 'LogQSearch:::' msg = { 'search_tag': search_tag, 'search_var': search_input, 'search_field': 'orig_message', 'search_oper': search_oper, 'time_min': time_min, 'time_max': time_max } msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT)) if search_type == 'D': search_tag = 'LogDSearch:::' msg = { 'search_tag': search_tag, 'search_var': search_input, 'search_field': 'orig_message', 'search_oper': search_oper, 'time_min': time_min, 'time_max': time_max } msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (DEEP_SCH_GRP, DEEP_SCH_PORT)) if search_type == 'W': search_tag = '!WRITECACHE!' msg = {'search_tag': search_tag} msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) if search_type == 'C': search_tag = '!SEARCHCACHE!' msg = { 'search_tag': search_tag, 'search_var': search_input, 'search_field': 'orig_message', 'search_oper': search_oper, 'api_file': api_file } msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) if search_type == 'ZD': search_tag = '!DELETEDISKCACHE!' msg = {'search_tag': search_tag} msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) if search_type == 'ZM': search_tag = '!DELETEMEMCACHE!' msg = {'search_tag': search_tag} msg = json.dumps(msg) msg = bytes(msg, "ascii") msg = cryp.EncryptMsg(msg, password) sock.sendto(msg, (SCH_GRP, SCH_PORT))
def searchDisk(dcdmsg, password, search_term_debug='off', response_limit=3000): with open('network_cfg.json', 'r') as nwc: nw = json.load(nwc) SCH_GRP = nw['SCH_GRP'] SCH_PORT = nw['SCH_PORT'] API_GRP = nw['API_GRP'] API_PORT = nw['API_PORT'] sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP) sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32) js_dcdmsg = json.loads(dcdmsg) so = js_dcdmsg['search_oper'] sv = js_dcdmsg['search_var'] sf = js_dcdmsg['search_field'] tmn = js_dcdmsg['time_min'] tmx = js_dcdmsg['time_max'] with open('localcfg.json', 'r') as lcf: lcfg = json.load(lcf) loglist = glob.glob('LOGS/msglog*.json') fldlist = [] search_result = [] for i in loglist: with open(i, 'r') as f: sch = json.load(f) for j in sch: fldlist.append(j) if so == 'OR': search_result = [j for j in fldlist if any(x in j[sf] for x in sv)] search_result = [ j for j in search_result if j["source_time"] > tmn and j["source_time"] < tmx ] if so == 'AND': search_result = [j for j in fldlist if all(x in j[sf] for x in sv)] search_result = [ j for j in search_result if j["source_time"] > tmn and j["source_time"] < tmx ] else: print("Using 'OR' operator") search_result = [j for j in fldlist if any(x in j[sf] for x in sv)] search_result = [ j for j in search_result if j["source_time"] > tmn and j["source_time"] < tmx ] if search_term_debug == 'on': print(so) print('is the search operator') print(sv) print('is search var\n') print(sf) print('is search field\n') print(search_result) print('is search result\n\n') print(len(search_result)) print('is the number of search results###\n\n') '''need to throttle responses using loop to limit msg size''' with open('nodelist_previous.json', 'r') as np: npl = json.load(np) lnp = len(npl) lsr = len(search_result) response_size = {'node_len': lnp, 'search_len': lsr} js_response_size = json.dumps(response_size) js_response_size = bytes(js_response_size, "ascii") msg = cryp.EncryptMsg(js_response_size, password) time.sleep(1) sock.sendto(msg, (API_GRP, API_PORT)) if lsr > response_limit: search_result = [{lcfg["localnode"]: lsr}] ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') print(st + ' is START time for sending deep search responses') i = 0 send_chunk = [] try: if search_result is not []: for k in search_result: send_chunk.append(k) i += 1 if i % 40 == 0: js_search_result = json.dumps(send_chunk) js_search_result = bytes(js_search_result, "ascii") msg = cryp.EncryptMsg(js_search_result, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) send_chunk = [] time.sleep(.2) if send_chunk != []: js_search_result = json.dumps(send_chunk) js_search_result = bytes(js_search_result, "ascii") msg = cryp.EncryptMsg(js_search_result, password) sock.sendto(msg, (SCH_GRP, SCH_PORT)) else: pass ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime( '%Y-%m-%d %H:%M:%S') print(st + ' is END time for sending deep search responses') except OSError: msg = ( "Attempting to send %s log messages from overran Tx buffer" % str(len(js_search_result))) pass return search_result