def _findAxonClone(self):
myhost = self.opts.get('hostname')
bytemax = self.opts.get('bytemax')
dyntask = s_common.gentask('info')
hostinfo = list(self.axonbus.callByTag('class.synapse.axon.AxonHost', dyntask))
hostinfo = [h for h in hostinfo if h[1].get('free') > bytemax]
hostinfo = [h for h in hostinfo if h[1].get('hostname') != myhost]
def hostkey(x):
used = x[1].get('used')
count = x[1].get('count')
return (count, used)
for svcfo, ahinfo in sorted(hostinfo, key=hostkey):
try:
host = ahinfo.get('hostname')
if host in self.clonehosts:
continue
props = {'clone': self.iden, 'bytemax': bytemax, 'host': host}
axfo = self.axonbus.callByIden(svcfo[0], 'add', **props)
tufo = self.core.formTufoByProp('axon:clone', axfo[0], host=host)
self.clonehosts.add(host)
return axfo
except Exception as e:
logger.exception('findAxonClone')
def find(self, htype, hvalu, bytag=axontag):
'''
Find and return any blobs with the given hash.
Example:
blobs = axon.find('sha256',valu)
'''
retblobs = []
dyntask = s_common.gentask('find', htype, hvalu)
for svcfo, blobs in self.svcprox.callByTag(bytag, dyntask):
if not blobs:
continue
try:
axon = self._getSvcAxon(svcfo[0])
if axon is None:
continue
[b[1].__setitem__('.axon', svcfo[0]) for b in blobs]
retblobs.extend(blobs)
except Exception as e:
logger.warning('AxonApi find: %s %s' % (svcfo[0], e))
return retblobs
def getTufosByPropFrom(self, prop, valu=None, limit=None, mintime=None, maxtime=None, fromtag=deftag):
dyntask = gentask('getTufosByFrob',prop,valu=valu,limit=limit,mintime=mintime,maxtime=maxtime)
for svcfo,retval in self.callByTag(fromtag,dyntask):
for tufo in retval:
tufo[1]['.from'] = svcfo[0]
yield tufo
def _oper_lift(self):
name,prop,valu = self.getByPropValu()
limt = self.kwargs.get('limit')
ftag = self.kwargs.get('from', s_opers_common.deftag)
dyntask = gentask('getTufosBy', name, prop, valu, limit=limt)
# TODO: timeouts
for svcfo,tufos in self.query.callByTag(ftag, dyntask):
for tufo in tufos:
tufo[1]['.from'] = svcfo[0]
self.query.add(tufo)
def _getTufosByFrom(self, by, prop, valu=None, limit=None, fromtag=None):
if fromtag == None:
fromtag = self.deftag
ret = []
limit = self.getLiftLimit(limit)
dyntask = s_common.gentask('stormTufosBy', by, prop, valu=valu, limit=limit)
for svcfo,retval in self.svcprox.callByTag(fromtag, dyntask, timeout=self.svctime):
[ tufo[1].__setitem__('.from',svcfo[0]) for tufo in retval ]
ret.extend(retval)
return ret
def bytes(self, htype, hvalu, bytag=axontag):
dyntask = s_common.gentask('find', htype, hvalu)
for svcfo, blobs in self.svcprox.callByTag(bytag, dyntask):
if not blobs:
continue
axon = self._getSvcAxon(svcfo[0])
if axon is None:
continue
for byts in axon.bytes(htype, hvalu):
yield byts
return
def has(self, htype, hvalu, bytag=axontag):
'''
Returns True if any of the axons in the cluster contain the given hash.
Example:
if not axapi.has('sha256',filehash):
dostuff()
'''
dyntask = s_common.gentask('has', htype, hvalu)
for svcfo, retval in self.svcprox.callByTag(bytag, dyntask):
if retval:
return True
return False
def byiden(self, iden, bytag=axontag):
'''
Get a axon:blob node by iden (superhash) value.
Args:
iden (str): Iden to look up.
Returns:
((str, dict)): Blob tufo returned by the Axon's cortex.
'''
dyntask = s_common.gentask('byiden', iden)
for svcfo, retval in self.svcprox.callByTag(bytag, dyntask):
if retval:
return retval
return None
def _oper_lift(self):
rtup = self.getLiftRange()
prop = self.args[0]
rtup = self.getLiftRange()
limt = self.kwargs.get('limit')
ftag = self.kwargs.get('from', s_opers_common.deftag)
dyntask = gentask('getTufosBy','range', prop, rtup, limit=limt)
# TODO: timeouts
for svcfo,tufos in self.query.callByTag(ftag, dyntask):
for tufo in tufos:
tufo[1]['.from'] = svcfo[0]
self.query.add(tufo)
def _findAxonClone(self):
myhost = self.getConfOpt('axon:hostname')
bytemax = self.getConfOpt('axon:bytemax')
dyntask = s_common.gentask('info')
hostinfo = list(
self.axonbus.callByTag('class.synapse.axon.AxonHost', dyntask))
hostinfo = [h for h in hostinfo if h[1].get('free') > bytemax]
hostinfo = [h for h in hostinfo if h[1].get('hostname') != myhost]
def hostkey(x):
used = x[1].get('used')
count = x[1].get('count')
return (count, used)
for svcfo, ahinfo in sorted(hostinfo, key=hostkey):
try:
host = ahinfo.get('hostname')
if host in self.clonehosts:
continue
props = {
'axon:clone': 1,
'axon:clones': 0,
'axon:clone:iden': self.iden,
'axon:bytemax': bytemax,
'axon:hostname': host,
}
axfo = self.axonbus.callByIden(svcfo[0], 'add', **props)
tufo = self.core.formTufoByProp('axon:clone',
axfo[0],
host=host)
self.clonehosts.add(host)
if not axfo: # pragma: no cover
logger.error(
'{} Did not get a clone for {} from {}'.format(
myhost, self.iden, host))
return axfo
except Exception as e:
logger.exception(
'Axon %s, svc iden %s, host %s, props %s (_findAxonClone)',
self.iden, svcfo[0], host, props)
def _getWrAxons(self, bytag=axontag):
wraxons = []
# FIXME cache this call for a few seconds
dyntask = s_common.gentask('getAxonInfo')
for svcfo, axfo in self.svcprox.callByTag(bytag, dyntask):
if axfo[1]['opts'].get('ro'):
continue
axon = self._getSvcAxon(svcfo[0])
if axon is None:
continue
wraxons.append(axon)
return wraxons
def getTufosByPropFrom(self,
prop,
valu=None,
limit=None,
mintime=None,
maxtime=None,
fromtag=deftag):
dyntask = gentask('getTufosByFrob',
prop,
valu=valu,
limit=limit,
mintime=mintime,
maxtime=maxtime)
for svcfo, retval in self.callByTag(fromtag, dyntask):
for tufo in retval:
tufo[1]['.from'] = svcfo[0]
yield tufo