def __init__(self, es_node="192.168.133.53", es_port="6775"): try: self.es_node = configParsed()['elasticSearch']['ip'] self.es_port = configParsed()['elasticSearch']['port'] except KeyError: self.es_port = es_port self.es_node = es_node self.es_client = self._get_es_client self.es_conn = self._es_connection(timeout)
def startInteractiveHunt(self): config = basicUtils.configParsed() #huntInputParams = config['defaultInputParams'] pipeLine = config['pipeLine'] pipeline = '' defaults = config['defaultInputParams'] for pipe, val in pipeLine.items(): self.count = self.count + 1 print("Working on pipeLine : %s " % pipe) print("Pipe Line Type : %s " % val['type']) print("Log to be taken : %s " % val['log']) print("") pipeline = map(lambda (x, y): y['huntName'], val['flow'].items()) for k, v in val['flow'].items(): try: defaults[v['huntName']][val['log']].update(v) except KeyError: raise Exception( "Default Params Not present for %s for hunt type : %s " % (val['log'], v['huntName'])) #pipeline = ':'.join(pipeline) if val['type'].lower() == 'interactive': print "Interactive Type of request made for the log : %s " % val[ 'log'] + val['prefix'] #val['flow'] #defaults[val['log']] self.runValidPipeline(val, defaults) elif val['type'].lower() == 'live': print "Live Type of request made for log : %s " % val[ 'log'] + val['prefix'] self.runLivePipe(val, defaults)
def get_permanent_datasources(self): mylist = [] config = configParsed() config = config['pipeLine'] config = map(lambda (x, y): y['log'] + y['prefix'], config.items()) for line in self.response_list: if 'averageKfps' in line['properties']: if line['name'] in config or True: mylist.append(line['name']) return mylist
def __init__(self): config = configParsed() self.redisPortHost = findKey('redisHostAndPort', config['pipeLine']) self.redisPortHost = list(self.redisPortHost) if len(self.redisPortHost) == 0: print "Taking redis connection info from default config" self.redisPortHost = list( findKey('redisHostAndPort', config['defaultInputParams']))[0] else: self.redisPortHost = self.redisPortHost[0] self.connect()
def __init__(self, ip='192.168.133.53', port='6773', json_folder=None): super (HuntApi, self).__init__ () self.tooldir = os.getcwd() if json_folder is None: self.json_folder = self.tooldir + "/Test_Data/json_folder/" self.headers = {"Content-Type": "application/json"} conf = configParsed() ip_port = conf['interactivehunt']['url'] self.url = ip_port self.current_request = None self.current_response = {} self.outputdataset = None self.driverId = None self.enrichment_name = None self.liveJson = []
def get_section(file, mysection='ingestion'): config = configParsed() return config[mysection] '''parser = ConfigParser.ConfigParser ()
def runLivePipe(self, pipe, params): #self.runValidPipeline(pipe,params) current_huntname = pipe['prefix'] + pipe['log'] + str( self.count) + "_" + "%05d" % randint(1, 99999) liveCallBody = self.runValidPipeline(pipe, params) dataset = pipe['log'] + pipe['prefix'] api = basicUtils.configParsed( )['liveHunt']['url'] + basicUtils.configParsed()['liveHunt']['api'] testObj = Enrichment() headers = basicUtils.configParsed()['liveHunt']['header'] result = testObj.execute_request("post", api, data=liveCallBody, headers=headers) outputDataSet = result.json()['output'] devobj = LaunchHunt() dataSetMetaInfo = Metadataservice().calculate_datasource_properties try: ST, ET = dataSetMetaInfo[outputDataSet][0:2] except KeyError: ST, ET = (-1, -1) print("Pushing the Logs:") config = basicUtils.configParsed() ingestionConfig = config['ingestion'] logPathServer = ingestionConfig['inputlogspath'] logPathLocal = ingestionConfig['localdatapath'] logType = pipe['log'] logPathLocal += '/' + logType output = subprocess.Popen('ls %s ' % logPathLocal, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).communicate() output = output[0].split('\n') output = map(lambda x: x.split('_'), output) output = map(lambda x: map(lambda y: y.split('.')[0], x), output) output = map(lambda x: x[1:], output) output = map(lambda x: ':'.join(x), output) pattern = '%d:%m:%Y:%H:%M' import time output = filter(lambda x: not x == '', output) epochOutput = map( lambda x: int(time.mktime(time.strptime(x, pattern))), output) output = map( lambda x: (int(time.mktime(time.strptime(x, pattern))), logType.upper() + '_' + '_'.join(x.split(':')) + '.log'), output) epochOutput = filter(lambda x: x > ET, epochOutput) output = filter(lambda (x, y): x in epochOutput, output) output = map(lambda (x, y): y, output) if len(output) == 0: raise Exception("No futher logs present for this") else: Ingestion().push_logs() while True: dataSetMetaInfo = Metadataservice().calculate_datasource_properties newET = dataSetMetaInfo[dataset][1] if newET > ET: break print "New Data Not yet injested Waiting for 60 sec" time.sleep(60) jsonBody = json.dumps( basicUtils.configParsed()['liveHunt']['pause']['jsonBody']) jsonBody['huntName'] = outputDataSet testObj.getsource(dataset, ST, ET) #Pause testObj.execute_request( 'post', basicUtils.configParsed()['liveHunt']['pause']['api'], basicUtils.configParsed()['liveHunt']['header'], data=jsonBody) devData = Metadataservice().get_static_dataset(outputDataSet) #Play jsonBody = json.dumps( basicUtils.configParsed()['liveHunt']['pause']['jsonBody']) jsonBody['huntName'] = outputDataSet testObj.execute_request( 'post', basicUtils.configParsed()['liveHunt']['play']['api'], basicUtils.configParsed()['liveHunt']['header'], data=jsonBody) for k, v in pipe['flow'].items(): qa_result = testObj.getHuntFunction(v['huntName'])( testObj, params[v['huntName']][pipe['log']]) comparator(devData, qa_result)