def displayJobs(**kwargs):
"""
"""
if kwargs['cmd'] == 'list':
atomFeed = rest.format.parseFeedDocument(kwargs['serverContent'])
if atomFeed:
print ASYNC_SEARCH_JOBS
for entry in atomFeed:
d = nodeToPrimitive(entry.rawcontents)
print '\tJob id: %s, ttl: %s\n' % (d['sid'], d['ttl'])
else:
print ASYNC_SEARCH_NONE
elif kwargs['cmd'] == 'remove':
if isinstance(kwargs['eaiArgsList']['jobid'], list):
#we have tried to do a remove jobs all
if kwargs['eaiArgsList']['jobid']:
print ASYNC_REMOVE_ALL % ','.join(kwargs['eaiArgsList']['jobid'])
else:
print ASYNC_SEARCH_NONE
else:
print 'Job id "%s" removed.' % kwargs['eaiArgsList']['jobid']
elif kwargs['cmd'] == 'show':
atom = rest.format.parseFeedDocument(kwargs['serverContent'])
d = nodeToPrimitive(atom.rawcontents)
print atom.title
print '-'*len(atom.title)
print '\n'.join(map(lambda x: '%s:%s' % (x[0], x[1]), d.items()))
elif kwargs['cmd'] == 'display':
try:
searchjob = getJob(kwargs['eaiArgsList']['jobid'], sessionKey=kwargs['sessionKey'])
except:
displayGenericError(cmd=kwargs['cmd'], obj='jobs', err_msg='Job id "%s" not found' % kwargs['eaiArgsList']['jobid'])
return
displaySyncSearch(searchjob=searchjob, **kwargs)
def getFieldInfo(args):
event = args['event']
job = se.getJob(args['sid'])
summary = job.summary
if event != None:
eventValues = {}
fieldValues = {}
fieldOrder = []
for attr in event:
if ignoredField(attr):
continue
fieldOrder.append(attr)
eventValues[attr] = [str(v) for v in event[attr]] # list
fieldValues[attr] = summary.fields.get(attr, {
'distinctCount': 0,
'modes': []
})
#fieldcount = fieldinfo['count']
#fielddistinctcount = fieldinfo['distinctCount']
#fieldValues[attr] = fieldinfo['modes'] # (vd['value'],vd['count'])
args['eventValues'] = eventValues
args['fieldValues'] = fieldValues
fieldOrder.sort(lambda x, y: fieldValues[y]['distinctCount'] -
fieldValues[x]['distinctCount'])
args['fieldOrder'] = fieldOrder
def getJobMessages(searchid, sessionKey):
try:
# try to get the job status immediately without retry
job = search.getJob(searchid, sessionKey=sessionKey, message_level="warn", status_fetch_timeout=0)
return job.messages
except Exception, e:
logger.error("Could not get job status for searchId=%s, Error: %s" % (searchid, str(e)))
def export(self):
job = search.getJob(self.sid, sessionKey=self.sessionKey)
kwargs = {
'output_mode': self.output_mode,
'count':
job.eventAvailableCount if self.count == 0 else self.count,
'offset': self.offset,
'field_list': self.field_list
}
data = streamJobExport(job, kwargs)
dispatch_dir = getDispatchDir(job.sid)
tmp_dir = os.path.join(dispatch_dir, 'export_tmp')
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
filename = '%s.%s' % (self.sid, self.output_mode)
self.local_path = os.path.join(tmp_dir, filename)
with open(self.local_path, 'w') as f:
for block in data:
f.write(str(block))
HadoopEnvManager.init(APP_NAME, 'nobody', self.sessionKey,
self.kerberos_principal)
cli_job = HadoopCliJob(HadoopEnvManager.getEnv(self.dst))
# attempt to make destination dir
cli_job.mkdir(self.dst)
cli_job.wait()
logger.error("[---] ExportResult export: Splunk Debug")
cli_job.moveFromLocal(self.local_path, self.dst + '/' + filename)
if cli_job.wait() != 0:
raise Exception("Failed to move file to hdfs: " +
cli_job.getOutput()[1])
def getEvent(sid, soffset, search, messages):
if sid == '' and soffset == '' and search == '':
return None
invalid = None
try:
if sid != '' and soffset != '':
job = se.getJob(sid)
return job.events[int(soffset)]
except Exception, e:
invalid = e
def getEvent(sid, soffset, search, messages):
if sid == '' and soffset == '' and search == '':
return None
invalid = None
try:
if sid != '' and soffset != '':
job = se.getJob(sid)
return job.events[int(soffset)]
except Exception, e:
invalid = e
def getJobInfo(requestArgs, messages):
try:
sid = requestArgs['sid']
offset = requestArgs['offset']
if sid != '' and offset != '':
job = se.getJob(sid)
return job.eventSearch, job.events[int(offset)]
except Exception, e:
addMessage(messages, 'Unable to get sample event. The search job has probably expired: "%s"' % e, CERROR)
return None, None
def getSampleEvents(eventtype, args, fast=True):
results = []
if eventtype != '':
if eventtype.strip().startswith("|") or len(shu.getJustCommands(eventtype, None)) > 1:
raise Exception("Eventtypes cannot contain search commands")
eventtype = eventtype.replace('\\', '\\\\')
sid = args['sid']
if fast:
# try to finalize jobs so that search job can be used with loadjob
try:
job = se.getJob(sid)
job.finalize()
se.waitForJob(job, MAX_JOB_WAIT) # job isn't ready immediately after finalize is called.
except Exception, ee:
pass
query = "| loadjob %s | search %s | head %s | fields | abstract maxlines=%s " % (sid, eventtype, MAX_SAMPLES, MAX_LINES)
else:
query = "search %s | head %s | fields | abstract maxlines=%s " % (eventtype, MAX_SAMPLES, MAX_LINES)
maxtime = args.get('maxtime', None)
if maxtime != None:
# try to use maxtime to get selecteed event at top
epochmaxtime = splunk.util.dt2epoch(splunk.util.parseISO(maxtime))
results = se.searchAll(query, latest_time=epochmaxtime, status_buckets=1,
auto_finalize_ec=MAX_SAMPLES,
max_out=MAX_SAMPLES,
max_count=MAX_SAMPLES, max_time=MAX_JOB_WAIT,
enable_lookups=0, auto_cancel=int(1.5*MAX_JOB_WAIT)
)
# if we got no results, perhaps the job expired. rerun the search.
if fast and len(results) == 0:
return getSampleEvents(eventtype, args, False)
# if not enough events, research without time constraint
if len(results) < MIN_SAMPLES:
results = se.searchAll(query, status_buckets=1,
auto_finalize_ec=MAX_SAMPLES,
max_out=MAX_SAMPLES,
max_count=MAX_SAMPLES, max_time=MAX_JOB_WAIT,
enable_lookups=0, auto_cancel=int(1.5*MAX_JOB_WAIT)
)
results = [ r.raw.getRaw() for r in results ]
def handleRemoveJobsAll(sessionKey, namespace, owner):
"""
current hack for removing all asyn jobs - to be removed when EAI endpoint gets written to do this...
"""
jobs = listJobs(sessionKey=sessionKey, namespace=namespace, owner=owner)
cancelled_jobs = []
for job in jobs:
j = getJob(job['sid'])
j.cancel()
cancelled_jobs.append(job['sid'])
#Call the appropriate display function...
try:
DISPLAY_CHARS['jobs'](cmd='remove', obj='jobs', eaiArgsList={'jobid':cancelled_jobs})
except KeyError, e:
logger.debug('endpoint: jobs')
logger.debug(str(e))
raise
def reset(self, **kwargs):
'''
Resets the user space to a clean state; usually used for testingm
'''
has_perms = True if 'admin'==au.getCurrentUser()['name'] else False
jobs_cancelled = []
if has_perms and cherrypy.request.method=='POST':
jobs = se.listJobs()
for job in jobs:
try:
j = se.getJob(job['sid'])
j.cancel()
jobs_cancelled.append(job['sid'])
except splunk.ResourceNotFound:
continue
return self.render_template('debug/reset.html', {
'has_perms': has_perms,
'method': cherrypy.request.method,
'jobs_cancelled': jobs_cancelled
})
def reset(self, **kwargs):
'''
Resets the user space to a clean state; usually used for testingm
'''
has_perms = True if 'admin'==au.getCurrentUser()['name'] else False
jobs_cancelled = []
if has_perms and cherrypy.request.method=='POST':
jobs = se.listJobs()
for job in jobs:
try:
j = se.getJob(job['sid'])
j.cancel()
jobs_cancelled.append(job['sid'])
except splunk.ResourceNotFound:
continue
return self.render_template('debug/reset.html', {
'has_perms': has_perms,
'method': cherrypy.request.method,
'jobs_cancelled': jobs_cancelled
})
def handleRemoveJobsAll(sessionKey, namespace, owner):
"""
current hack for removing all asyn jobs - to be removed when EAI endpoint gets written to do this...
"""
jobs = listJobs(sessionKey=sessionKey, namespace=namespace, owner=owner)
cancelled_jobs = []
for job in jobs:
j = getJob(job['sid'])
j.cancel()
cancelled_jobs.append(job['sid'])
#Call the appropriate display function...
try:
DISPLAY_CHARS['jobs'](cmd='remove',
obj='jobs',
eaiArgsList={
'jobid': cancelled_jobs
})
except KeyError, e:
logger.debug('endpoint: jobs')
logger.debug(str(e))
raise
def getFieldInfo(args):
event = args['event']
job = se.getJob(args['sid'])
summary = job.summary
if event != None:
eventValues = {}
fieldValues = {}
fieldOrder = []
for attr in event:
if ignoredField(attr):
continue
fieldOrder.append(attr)
eventValues[attr] = [str(v) for v in event[attr]] # list
fieldValues[attr] = summary.fields.get(attr,{'distinctCount':0, 'modes':[]})
#fieldcount = fieldinfo['count']
#fielddistinctcount = fieldinfo['distinctCount']
#fieldValues[attr] = fieldinfo['modes'] # (vd['value'],vd['count'])
args['eventValues'] = eventValues
args['fieldValues'] = fieldValues
fieldOrder.sort(lambda x, y: fieldValues[x]['distinctCount'] - fieldValues[y]['distinctCount'])
args['fieldOrder'] = fieldOrder
def getJobMessages(searchid, sessionKey):
try:
job = search.getJob(searchid, sessionKey=sessionKey, message_level='warn')
return job.messages
except Exception, e:
logger.error("Could not get job status for searchId=%s, Error: %s" % (searchid, str(e)))
def generatePDF(serverURL, subject, sid, settings, pdfViewID, ssName,
paperSize, paperOrientation):
"""
Reach out and retrieve a PDF copy of the search results if possible
and return the MIME attachment
"""
sessionKey = settings.get('sessionKey', None)
owner = settings.get('owner', 'nobody')
if not sessionKey:
raise PDFException("Can't attach PDF - sessionKey unavailable")
# build up filename to use with attachments
pdfViewID_filename = pdfViewID and pdfViewID.strip(' .:;|><\'"')
datestamp = time.strftime('%Y-%m-%d')
if pdfViewID_filename:
filename = '%s-%s.pdf' % (pdfViewID_filename[:50], datestamp)
# strip control characters, forward & backslash
filename = re.sub(r'[\x00-\x1f\x7f/\\]+', '-', filename)
if isinstance(filename, unicode):
filename = filename.encode(CHARSET)
else:
filename = 'splunk-report-%s.pdf' % datestamp
# build up parameters to the PDF server
parameters = {}
parameters['namespace'] = settings["namespace"]
parameters['owner'] = owner
if pdfViewID:
parameters['input-dashboard'] = pdfViewID
else:
if ssName:
parameters['input-report'] = ssName
elif sid:
# in the event where sendemail is called from search
# and we need to generate pdf re-run the search
job = search.getJob(sid, sessionKey=sessionKey)
jsonJob = job.toJsonable(timeFormat='unix')
searchToRun = jsonJob.get('search').strip()
if searchToRun.lower().startswith('search '):
searchToRun = searchToRun[7:]
sendemailRegex = r'\|\s*sendemail'
if (re.findall(sendemailRegex, searchToRun)):
parameters['input-search'] = re.split(sendemailRegex,
searchToRun)[0]
parameters['et'] = jsonJob.get('earliestTime')
parameters['lt'] = jsonJob.get('latestTime')
else:
raise PDFException(
"Can't attach PDF - ssName and pdfViewID unavailable")
if sid:
if type(sid) is dict:
for sidKey in sid:
parameters[sidKey] = sid[sidKey]
else:
parameters['sid'] = sid
if paperSize and len(paperSize) > 0:
if paperOrientation and paperOrientation != "portrait":
parameters['paper-size'] = "%s-%s" % (paperSize, paperOrientation)
else:
parameters['paper-size'] = paperSize
# determine if we should set an effective dispatch "now" time for this job
scheduledJobEffectiveTime = getEffectiveTimeOfScheduledJob(
settings.get("sid", ""))
logger.info("sendemail:mail effectiveTime=%s" % scheduledJobEffectiveTime)
if scheduledJobEffectiveTime != None:
parameters['now'] = scheduledJobEffectiveTime
try:
# Ensure compatibility with systems with supportSSLV3Only=tru
setSSLWrapProtocol(
ssl.PROTOCOL_SSLv3
) #not sure we need this now that we are using simpleRequest instead of urlopen
response, content = simpleRequest(
"pdfgen/render",
sessionKey=sessionKey,
getargs=parameters,
timeout=PDFGEN_SIMPLE_REQUEST_TIMEOUT)
except splunk.SplunkdConnectionException, e:
raise PDFException(
"Failed to fetch PDF (SplunkdConnectionException): %s" % str(e))
def generatePDF(serverURL, subject, sid, settings, pdfViewID, ssName, paperSize, paperOrientation):
"""
Reach out and retrieve a PDF copy of the search results if possible
and return the MIME attachment
"""
sessionKey = settings.get('sessionKey', None)
owner = settings.get('owner', 'nobody')
if not sessionKey:
raise PDFException("Can't attach PDF - sessionKey unavailable")
# build up filename to use with attachments
pdfViewID_filename = pdfViewID and pdfViewID.strip(' .:;|><\'"')
datestamp = time.strftime('%Y-%m-%d')
if pdfViewID_filename:
filename = '%s-%s.pdf' % (pdfViewID_filename[:50], datestamp)
# strip control characters, forward & backslash
filename = re.sub(r'[\x00-\x1f\x7f/\\]+', '-', filename)
if isinstance(filename, unicode):
filename = filename.encode(CHARSET)
else:
filename = 'splunk-report-%s.pdf' % datestamp
# build up parameters to the PDF server
parameters = {}
parameters['namespace'] = settings["namespace"]
parameters['owner'] = owner
if pdfViewID:
parameters['input-dashboard'] = pdfViewID
else:
if ssName:
parameters['input-report'] = ssName
elif sid:
# in the event where sendemail is called from search
# and we need to generate pdf re-run the search
job = search.getJob(sid, sessionKey=sessionKey)
jsonJob = job.toJsonable(timeFormat='unix')
searchToRun = jsonJob.get('search').strip()
if searchToRun.lower().startswith('search '):
searchToRun = searchToRun[7:]
sendemailRegex = r'\|\s*sendemail'
if (re.findall(sendemailRegex, searchToRun)):
parameters['input-search'] = re.split(sendemailRegex, searchToRun)[0]
parameters['et'] = jsonJob.get('earliestTime')
parameters['lt'] = jsonJob.get('latestTime')
else:
raise PDFException("Can't attach PDF - ssName and pdfViewID unavailable")
if sid:
if type(sid) is dict:
for sidKey in sid:
parameters[sidKey] = sid[sidKey]
else:
parameters['sid'] = sid
if paperSize and len(paperSize) > 0:
if paperOrientation and paperOrientation != "portrait":
parameters['paper-size'] = "%s-%s" % (paperSize, paperOrientation)
else:
parameters['paper-size'] = paperSize
# determine if we should set an effective dispatch "now" time for this job
scheduledJobEffectiveTime = getEffectiveTimeOfScheduledJob(settings.get("sid", ""))
logger.info("sendemail:mail effectiveTime=%s" % scheduledJobEffectiveTime)
if scheduledJobEffectiveTime != None:
parameters['now'] = scheduledJobEffectiveTime
try:
# Ensure compatibility with systems with supportSSLV3Only=tru
setSSLWrapProtocol(ssl.PROTOCOL_SSLv3) #not sure we need this now that we are using simpleRequest instead of urlopen
response, content = simpleRequest("pdfgen/render", sessionKey = sessionKey, getargs = parameters, timeout = PDFGEN_SIMPLE_REQUEST_TIMEOUT)
except splunk.SplunkdConnectionException, e:
raise PDFException("Failed to fetch PDF (SplunkdConnectionException): %s" % str(e))
