def getSguilSql(query, sguilserver=None, serverUser=None, serverPass=None, dbuser='******', dbpass=None, sguildb='securityonion_db', tableSplit=False): if serverUser: sguilDBServer = initSSH(sguilserver, user=serverUser, pwd=serverPass, pubpriv=False) else: sguilDBServer = initSSH(sguilserver) if not query.endswith(';'): query = query + ';' if dbpass: stdin, stdout, stderr = sguilDBServer.exec_command('''mysql -u %s --password=%s -D %s -e "%s"''' % (dbuser, dbpass, sguildb, query)) else: stdin, stdout, stderr = sguilDBServer.exec_command('''mysql -u %s -D %s -e "%s"''' % (dbuser, sguildb, query)) if tableSplit: return [x.split('\t') for x in stdout.read().splitlines()] else: return stdout.read() err = stderr.read() if err: print(err)
def execute(event): if not event.pcaps: event.setAttribute('pcaps', []) for server in [x.strip() for x in confVars.so_sensors.split(',')]: ssh = initSSH(server) dailies = getDailylogsInScope(event, ssh) tcpdumpFiles(event, ssh, server, dailies) if confVars.mergeGroups: mergePCAPGroups(event)
def pullGenericDaily(self, egrepInclude, egrepExclude, startDate, endDate, server, logpath, outputExtension, compressionDelay, compressionExtension, toFile=True, toStdOut=False, customCmd=None, append=False, retResults=False): if customCmd: cmd = customCmd elif not egrepExclude: cmd = 'egrep "%s"' % (egrepInclude) else: cmd = 'egrep "%s" | egrep -v "%s"' % (egrepInclude, egrepExclude) today = datetime.datetime.today() oneDay = datetime.timedelta(days=1) fileName = os.path.basename(logpath) server = initSSH(server, event=self.event) tick = time() output = "" while startDate <= endDate: daysBack = (today.date() - startDate.date()).days - 1 if today.date() != datetime.datetime.today().date(): print('WARNING: it appears the log pull was still in progress beyond midnight.') print('There is a strong likelihood that log data will be missed.') if daysBack == -1: #if not toStdOut or collect: print("Checking %s..." % (fileName)) fullcmd = 'cat %s | %s' % (logpath, cmd) stdin, stdout, stderr = server.exec_command(fullcmd) #print stderr.readlines() elif daysBack > compressionDelay: if compressionExtension == 'bz2': #if not toStdOut or collect: print("Checking %s.%s.%s..." % (fileName, str(daysBack), compressionExtension)) fullcmd = 'bzcat %s.%s.%s | %s' % (logpath, str(daysBack), compressionExtension, cmd) stdin, stdout, stderr = server.exec_command(fullcmd) #print stderr.readlines() else: '''To add additional compression functionality to the daily pull please elif this block and add your logic here.''' logging.warning('The daily logs you are attempting to pull are in a compression format, %s, that is currently not implemented' % (compressionExtension)) else: #if not toStdOut or collect: print("Checking %s.%s..." % (fileName, str(daysBack))) fullcmd = 'cat %s.%s | %s' % (logpath, str(daysBack), cmd) stdin, stdout, stderr = server.exec_command(fullcmd) #print stderr.readlines() startDate += oneDay output += stdout.read() if not toStdOut: print('\n\nRetrieval time: %s\n' % (str(time() - tick))) if retResults: return output
def pullDaily(self, egrepInclude, egrepExclude, startDate, endDate, server, logpath, outputExtension, compressionDelay, compressionExtension, formalName, toFile=True, toStdOut=False, collect=False, formatter=None, header=None, customCmd=None, append=False, retResults=False): print("Pulling logs from %s to %s\n" % (startDate.date().isoformat(), endDate.date().isoformat())) if customCmd: cmd = customCmd elif not egrepExclude: cmd = 'egrep "%s"' % (egrepInclude) else: cmd = 'egrep "%s" | egrep -v "%s"' % (egrepInclude, egrepExclude) print('command: ... | %s\n' % cmd) today = datetime.datetime.today() oneDay = datetime.timedelta(days=1) fileName = os.path.basename(logpath) server = initSSH(server, event=self.event) tick = time() output = "" orf = '%s.%s' % (self.event._baseFilePath, outputExtension) if append: outRawFile = open(orf, 'a') else: outRawFile = open(orf, 'w') if formatter: off = '%s.%s' % (self.event._baseFilePath, outputExtension + 'f') if append: outFileFormatted = open(off, 'a') else: outFileFormatted = open(off, 'w') if header and toStdOut and not collect: print(header) while startDate <= endDate: currentDate = startDate.strftime("%Y%m%d") if today.date() != datetime.datetime.today().date(): print('WARNING: it appears the log pull was still in progress beyond midnight.') print('There is a strong likelihood that log data will be missed.') if (today.date() - startDate.date()).days >= int(compressionDelay): if compressionExtension in ['bz2', 'gz', 'xz']: if compressionExtension == 'bz2': catCmd = 'bzcat' elif compressionExtension == 'gz': catCmd = 'zcat' elif compressionExtension == 'xz': catCmd = 'xzcat' if not toStdOut or collect: print("Checking %s.%s.log.%s..." % (fileName, currentDate, compressionExtension)) fullcmd = '%s %s.%s.log.%s | %s' % (catCmd, logpath, currentDate, compressionExtension, cmd) logging.debug('msg="Complete pull command" fullcmd="%s"' % fullcmd) stdin, stdout, stderr = server.exec_command(fullcmd) #print stderr.readlines() else: '''To add additional compression functionality to the daily pull please elif this block and add your logic here.''' logging.warning('The daily logs you are attempting to pull are in a compression format, %s, that is currently not implemented' % (compressionExtension)) else: if not toStdOut or collect: print("Checking %s.%s.log..." % (fileName, currentDate)) fullcmd = 'cat %s.%s.log | %s' % (logpath, currentDate, cmd) logging.debug('msg="Complete pull command" fullcmd="%s"' % fullcmd) stdin, stdout, stderr = server.exec_command(fullcmd) #print stderr.readlines() startDate += oneDay if collect: output += stdout.read() #output += 'tada\n' else: out = stdout.read() #out = 'tada' if formatter: outf = formatter(out) outFileFormatted.write(outf) outFileFormatted.flush() if toFile: outRawFile.write(out) outRawFile.flush() if toStdOut: if formatter: print(outf) else: print(out) if collect: if formatter: outf = formatter(output) outFileFormatted.write(outf) outFileFormatted.flush() if toFile: outRawFile.write(output) outRawFile.flush() if toStdOut: print('\n\nRetrieval time: %s\n' % (str(time() - tick))) print('') if header: print(header) if formatter: print(outf) else: print(output) if formatter: outFileFormatted.close() outRawFile.close() if not toStdOut: print('\n\nRetrieval time: %s\n' % (str(time() - tick))) if formatter: print('\nFormatted %s results saved to: %s' % (formalName, off)) print('Raw %s results saved to: %s' % (formalName, orf)) elif toFile: print('\nRaw %s results saved to: %s' % (formalName, orf)) print('') if retResults: return output else: return orf