def logMonitor_Tx(msg, params, password, nw):
LOGMSG_GRP = nw['LOGMSG_GRP']
LOGMSG_PORT = nw['LOGMSG_PORT']
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
print("Starting Tx process...\n")
localnode = params['localnode']
role = params['role']
node = params['localnode']
cluster = params['cluster_id']
hostname = (socket.gethostname())
jobs = []
z = multiprocessing.Process(target=nm.infHBeat, args=(params, nw))
jobs.append(z)
z.daemon = True
z.start()
n = multiprocessing.Process(target=nm.messageTagGen, args=(nw, ))
jobs.append(n)
n.daemon = True
n.start()
if role == 'RxTx':
p = multiprocessing.Process(target=logMonitor_Rx,
args=(
password,
params,
))
jobs.append(p)
p.daemon = True
p.start()
ds = multiprocessing.Process(target=sl.deepSearch)
jobs.append(ds)
ds.daemon = True
ds.start()
q = multiprocessing.Process(target=syslog)
jobs.append(q)
q.daemon = True
q.start()
lenfr = 0
send_throttle = 2
lfr = [0, 0]
while True:
lfr[0] = lfr[1]
if max(lfr) > 100:
with open('syslog.log', 'w') as f:
f.close()
lfr = [0, 0]
time.sleep(send_throttle)
try:
with open('droplist.json', 'r') as dlj:
drop_tag = json.load(dlj)
drop_tag = str(drop_tag)
except:
print('possible JSONDecodeError')
drop_tag = '[]'
pass
while True:
with open('syslog.log', 'r') as f:
fr = f.readlines()
lfr[1] = len(fr)
if lfr[1] > lfr[0]:
msg = ''
for i in fr[lfr[0]:lfr[1]]:
msg = i.rstrip()
parse_msg = json.loads(msg)
ts = time.time()
msg = {
'source_time': ts,
'sending_node': localnode,
'sending_hostname': hostname,
'orig_message': parse_msg['log_message'],
'orig_addr': parse_msg['orig_addr'],
'drop_tag': drop_tag
}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
try:
sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT))
except OSError:
msg = (
"Attempting to send %s log messages from overran Tx buffer"
% str(len(fr)))
msg = localnode + '@' + hostname + "# " + '"' + msg + '"' + drop_tag
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT))
pass
if lfr[0] == lfr[1]:
pass
else:
pass
break
sys.exit()
def searchMem(search_list, dcdmsg, password, search_term_debug='off'):
with open('network_cfg.json', 'r') as nwc:
nw = json.load(nwc)
SCH_GRP = nw['SCH_GRP']
SCH_PORT = nw['SCH_PORT']
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
js_dcdmsg = json.loads(dcdmsg)
so = js_dcdmsg['search_oper']
sv = js_dcdmsg['search_var']
sf = js_dcdmsg['search_field']
tmn = js_dcdmsg['time_min']
tmx = js_dcdmsg['time_max']
search_result = []
if so == 'OR':
search_result = [
j for j in search_list if any(x in j[sf] for x in sv)
]
search_result = [
j for j in search_result
if j["source_time"] > tmn and j["source_time"] < tmx
]
if so == 'AND':
search_result = [
j for j in search_list if all(x in j[sf] for x in sv)
]
else:
print("Using 'OR' operator")
search_result = [
j for j in search_list if any(x in j[sf] for x in sv)
]
search_result = [
j for j in search_result
if j["source_time"] > tmn and j["source_time"] < tmx
]
if search_term_debug == 'on':
print(so)
print('is the search operator')
print(sv)
print('is search var\n')
print(sf)
print('is search field\n')
print(search_result)
print('is search result\n\n')
print(len(search_result))
print('is the number of search results###\n\n')
'''need to throttle responses using loop to limit msg size'''
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
print(st + ' is START time for sending memory search responses')
i = 0
send_chunk = []
try:
if search_result is not []:
for k in search_result:
send_chunk.append(k)
i += 1
if i % 40 == 0:
js_search_result = json.dumps(send_chunk)
js_search_result = bytes(js_search_result, "ascii")
msg = cryp.EncryptMsg(js_search_result, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
send_chunk = []
time.sleep(.2)
if send_chunk != []:
js_search_result = json.dumps(send_chunk)
js_search_result = bytes(js_search_result, "ascii")
msg = cryp.EncryptMsg(js_search_result, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
else:
pass
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d %H:%M:%S')
print(st + ' is END time for sending memory search responses')
except OSError:
msg = (
"Attempting to send %s log messages from overran Tx buffer" %
str(len(js_search_result)))
msg = localnode + '@' + hostname + "# " + '"' + msg + '"' + drop_tag
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
pass
return search_result
def searchSend(input_type='user',
search_input=[],
search_oper='AND',
search_type='Q',
api_file=None,
time_min=0,
time_max=30000000000):
with open('pwf', 'r') as p:
password = p.read()
password = password.rstrip()
with open('network_cfg.json', 'r') as nwc:
nw = json.load(nwc)
LOGMSG_GRP = nw['LOGMSG_GRP']
LOGMSG_PORT = nw['LOGMSG_PORT']
SCH_GRP = nw['SCH_GRP']
SCH_PORT = nw['SCH_PORT']
DEEP_SCH_GRP = nw['DEEP_SCH_GRP']
DEEP_SCH_PORT = nw['DEEP_SCH_PORT']
'''
may want to break out cache search to help with scaling
SEARCH_CACHE_GRP='232.1.1.12'
SEARCH_CACHE_PORT = 5012
'''
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
if input_type == 'user':
in_field = ''
search_input = []
while '!!!' not in in_field:
in_field = str(
input("Please enter search string, enter !!! to end: "))
search_input.append(in_field)
search_input.pop()
print(search_input)
search_oper = str(
input("Enter your search operator as AND , OR -> "))
search_type = str(
input(
"Enter search type Q:Quick, D:Deep:, C:Cached or ZM/ZD:Zero-Mem/Disk-Cache,W:Write-Cache -> "
))
if search_type == 'Q':
search_tag = 'LogQSearch:::'
msg = {
'search_tag': search_tag,
'search_var': search_input,
'search_field': 'orig_message',
'search_oper': search_oper,
'time_min': time_min,
'time_max': time_max
}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (LOGMSG_GRP, LOGMSG_PORT))
if search_type == 'D':
search_tag = 'LogDSearch:::'
msg = {
'search_tag': search_tag,
'search_var': search_input,
'search_field': 'orig_message',
'search_oper': search_oper,
'time_min': time_min,
'time_max': time_max
}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (DEEP_SCH_GRP, DEEP_SCH_PORT))
if search_type == 'W':
search_tag = '!WRITECACHE!'
msg = {'search_tag': search_tag}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
if search_type == 'C':
search_tag = '!SEARCHCACHE!'
msg = {
'search_tag': search_tag,
'search_var': search_input,
'search_field': 'orig_message',
'search_oper': search_oper,
'api_file': api_file
}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
if search_type == 'ZD':
search_tag = '!DELETEDISKCACHE!'
msg = {'search_tag': search_tag}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
if search_type == 'ZM':
search_tag = '!DELETEMEMCACHE!'
msg = {'search_tag': search_tag}
msg = json.dumps(msg)
msg = bytes(msg, "ascii")
msg = cryp.EncryptMsg(msg, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
def searchDisk(dcdmsg,
password,
search_term_debug='off',
response_limit=3000):
with open('network_cfg.json', 'r') as nwc:
nw = json.load(nwc)
SCH_GRP = nw['SCH_GRP']
SCH_PORT = nw['SCH_PORT']
API_GRP = nw['API_GRP']
API_PORT = nw['API_PORT']
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM,
socket.IPPROTO_UDP)
sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 32)
js_dcdmsg = json.loads(dcdmsg)
so = js_dcdmsg['search_oper']
sv = js_dcdmsg['search_var']
sf = js_dcdmsg['search_field']
tmn = js_dcdmsg['time_min']
tmx = js_dcdmsg['time_max']
with open('localcfg.json', 'r') as lcf:
lcfg = json.load(lcf)
loglist = glob.glob('LOGS/msglog*.json')
fldlist = []
search_result = []
for i in loglist:
with open(i, 'r') as f:
sch = json.load(f)
for j in sch:
fldlist.append(j)
if so == 'OR':
search_result = [j for j in fldlist if any(x in j[sf] for x in sv)]
search_result = [
j for j in search_result
if j["source_time"] > tmn and j["source_time"] < tmx
]
if so == 'AND':
search_result = [j for j in fldlist if all(x in j[sf] for x in sv)]
search_result = [
j for j in search_result
if j["source_time"] > tmn and j["source_time"] < tmx
]
else:
print("Using 'OR' operator")
search_result = [j for j in fldlist if any(x in j[sf] for x in sv)]
search_result = [
j for j in search_result
if j["source_time"] > tmn and j["source_time"] < tmx
]
if search_term_debug == 'on':
print(so)
print('is the search operator')
print(sv)
print('is search var\n')
print(sf)
print('is search field\n')
print(search_result)
print('is search result\n\n')
print(len(search_result))
print('is the number of search results###\n\n')
'''need to throttle responses using loop to limit msg size'''
with open('nodelist_previous.json', 'r') as np:
npl = json.load(np)
lnp = len(npl)
lsr = len(search_result)
response_size = {'node_len': lnp, 'search_len': lsr}
js_response_size = json.dumps(response_size)
js_response_size = bytes(js_response_size, "ascii")
msg = cryp.EncryptMsg(js_response_size, password)
time.sleep(1)
sock.sendto(msg, (API_GRP, API_PORT))
if lsr > response_limit:
search_result = [{lcfg["localnode"]: lsr}]
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
print(st + ' is START time for sending deep search responses')
i = 0
send_chunk = []
try:
if search_result is not []:
for k in search_result:
send_chunk.append(k)
i += 1
if i % 40 == 0:
js_search_result = json.dumps(send_chunk)
js_search_result = bytes(js_search_result, "ascii")
msg = cryp.EncryptMsg(js_search_result, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
send_chunk = []
time.sleep(.2)
if send_chunk != []:
js_search_result = json.dumps(send_chunk)
js_search_result = bytes(js_search_result, "ascii")
msg = cryp.EncryptMsg(js_search_result, password)
sock.sendto(msg, (SCH_GRP, SCH_PORT))
else:
pass
ts = time.time()
st = datetime.datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d %H:%M:%S')
print(st + ' is END time for sending deep search responses')
except OSError:
msg = (
"Attempting to send %s log messages from overran Tx buffer" %
str(len(js_search_result)))
pass
return search_result
