def get_virk_search(from_date):
client = elasticsearch1.Elasticsearch('http://distribution.virk.dk:80',
timeout=300)
s = Search(using=client, index='offentliggoerelser')
s = s.filter('range', offentliggoerelsesTidspunkt={'gte': from_date})
s = s.sort('offentliggoerelsesTidspunkt')
return s
def search(time_s, time_e, fwd_host_list, index):
time_start = time_s[:10] +'T' + time_s[11:] + '.000Z'
time_end =time_e[:10] + 'T' + time_e[11:] + '.000Z'
index_all = "logstash-" + index
ES_SERVERS = [{'host' : ES_HOST, 'port': 9200}]
es_client = ES.Elasticsearch(hosts = ES_SERVERS)
match_query = []
for i in xrange(len(fwd_host_list)):
match_query.append({"match":{"host": fwd_host_list[i]}})
es_search_options = {"query":{"bool": {"must":[{"range":{"@timestamp":{"gt":time_start,"le":time_end}}}]}}}
es_result = helpers.scan(
client = es_client,
query = es_search_options,
scroll = '3m',
index = index_all,
doc_type = 'redis-input',
timeout = '1m'
)
final_result_host = []
final_result_message = []
final_result_time = []
rlen = 0
for item in es_result:
rlen += 1
final_result_message.append(str(item['_source']['message']))
final_result_host.append(str(item['_source']['host']))
final_result_time.append(str(item['_source']['@timestamp']))
return final_result_message,final_result_time, final_result_host
def search(time_s, time_e, host, index, host_t):
time_start = time_s[:10] + "T" + time_s[11:] + ".000Z"
time_end = time_e[:10] + "T" + time_e[11:] + ".000Z"
#print time_start," ",time_end
host_all = "20.0.8." + str(host_t)
ES_SERVERS = [{'host': host_all, 'port': 9200}]
global es_client
es_client = ES.Elasticsearch(hosts=ES_SERVERS)
index_all = "logstash-" + index
es_search_options = set_search_optional(time_start, time_end, host)
es_result = get_search_result(es_search_options, index_all)
final_result_message, final_result_time = get_result_list(es_result)
return final_result_message, final_result_time
def search(es_stime, es_etime, ip_list, index):
index_all = 'logstash-' + index
es_servers = [{'host':ES_HOST, 'port':9200}]
match_query = []
for i in range(len(ip_list)):
match_query.append({'match':{'host':ip_list[i]}})
es_search_options = {'query':{'bool':{
'must':[{'bool':{'should':[match_query]}},{'range':{'@timestamp':{'gt':es_stime, 'lt':es_etime}}}],
'should':[{'match':{'message':'OPEN'}},{'match':{'message':'RELEASE'}},
{'match':{'message':'READ'}},{'match':{'message':'WRITE'}}]
}}}
es_client = ES.Elasticsearch(hosts = es_servers)
es_result = helpers.scan(client = es_client, query = es_search_options, scroll = '5m', index = index_all, doc_type = 'redis-input', timeout = '1m')
res_message = []
for item in es_result:
res_message.append((item['_source']['@timestamp'][0:10] + ' ' + item['_source']['@timestamp'][11:19] + ' ' + item['_source']['message'][24:] + ' ' + item['_source']['host']).split(' '))
return res_message
def search(time_s, time_e, index, host_t):
time_start = time_s[:10] + 'T' + time_s[11:] + '.000Z'
time_end = time_e[:10] + 'T' + time_e[11:] + '.000Z'
host_all = '20.0.8.' + str(host_t)
index_all = "logstash-" + index[0]
ES_SERVERS = [{'host': host_all, 'port': 9200}]
es_client = ES.Elasticsearch(hosts=ES_SERVERS)
es_search_options = {
"query": {
"bool": {
"must": [{
"range": {
"@timestamp": {
"gt": time_start,
"lt": time_end
}
}
}]
}
}
}
es_result = helpers.scan(client=es_client,
query=es_search_options,
scroll='3m',
index=index_all,
doc_type='redis-input',
timeout='1m')
final_result_message = []
final_result_time = []
rlen = 0
for item in es_result:
rlen += 1
final_result_message.append(str(item['_source']['message']))
final_result_time.append(str(item['_source']['@timestamp']))
print "RESULT LEN : " + str(rlen)
return final_result_message, final_result_time
#!/tps/bin/python -B
import os, sys, json, re
from math import floor
import elasticsearch1, urllib3
from elasticsearch1 import helpers
pwd = os.getcwd()
sys.path.insert(0, '{}/msl-datalytics/src/'.format(pwd))
from spazz import *
import timeit
start = time.time()
# from msldatalytics.src.spazz import *
#from spazz import *
es = elasticsearch1.Elasticsearch('https://msl-ops-es.cld.jpl.nasa.gov',
sniff_on_start=False)
# es = elasticsearch1.Elasticsearch('https://msl-ops-es.cld.jpl.nasa.gov',sniff_on_start=False)
urllib3.disable_warnings()
global index
index = 'mslice_db'
def main():
#Query for all submasters. We want all activity groups (Pie observations) where the seqID field = sub_XXXX in the last 1000 sols.
# --------------------------------------------- Input Parameters and Initializaton -------------------------------------------------
# parameters that should eventually be inputs
verbose = False # a verbose flag that identifies every time a submaster was rejected from the analysis
filename = 'demonstrationoutput' # name of the .json file output to be used as a pseudo-database
queryLen = 5000 # how large do we let the query get. Currently we wouldn't want anything larger than 5000 results
# earliestSol = 2170 # the earliest sol of results we want to include in our data. With our naming convention for submaster sequences we should only query within modulo 1000
