def test_model_inet_srv4_types(self):
with s_cortex.openurl('ram:///') as core:
core.setConfOpt('enforce', 1)
t0 = core.formTufoByProp('inet:tcp4', '8.8.8.8:80')
form, pprop = s_tufo.ndef(t0)
self.eq(pprop, 8830587502672)
self.eq(t0[1].get('inet:tcp4:port'), 80)
self.eq(t0[1].get('inet:tcp4:ipv4'),
core.getTypeNorm('inet:ipv4', '8.8.8.8')[0])
# 1.2.3.4:8443
t1 = core.formTufoByProp('inet:tcp4', 1108152164603)
self.eq(t1[1].get('inet:tcp4:port'), 8443)
self.eq(t1[1].get('inet:tcp4:ipv4'),
core.getTypeNorm('inet:ipv4', '1.2.3.4')[0])
t2 = core.formTufoByProp('inet:udp4', '8.8.8.8:80')
form, pprop = s_tufo.ndef(t2)
self.eq(pprop, 8830587502672)
self.eq(t2[1].get('inet:udp4:port'), 80)
self.eq(t2[1].get('inet:udp4:ipv4'),
core.getTypeNorm('inet:ipv4', '8.8.8.8')[0])
# 1.2.3.4:8443
t3 = core.formTufoByProp('inet:udp4', 1108152164603)
self.eq(t3[1].get('inet:udp4:port'), 8443)
self.eq(t3[1].get('inet:udp4:ipv4'),
core.getTypeNorm('inet:ipv4', '1.2.3.4')[0])
def test_model_inet_srv6_types(self):
with s_cortex.openurl('ram:///') as core:
core.setConfOpt('enforce', 1)
t0 = core.formTufoByProp('inet:tcp6', '[0:0:0:0:0:0:0:1]:80')
form, pprop = s_tufo.ndef(t0)
self.eq(pprop, '[::1]:80')
self.eq(t0[1].get('inet:tcp6:port'), 80)
self.eq(t0[1].get('inet:tcp6:ipv6'), '::1')
t1 = core.formTufoByProp('inet:tcp6', '[0:0:0:0:0:3:2:1]:443')
form, pprop = s_tufo.ndef(t1)
self.eq(pprop, '[::3:2:1]:443')
self.eq(t1[1].get('inet:tcp6:port'), 443)
self.eq(t1[1].get('inet:tcp6:ipv6'), '::3:2:1')
t2 = core.formTufoByProp('inet:udp6', '[0:0:0:0:0:3:2:1]:5000')
form, pprop = s_tufo.ndef(t2)
self.eq(pprop, '[::3:2:1]:5000')
self.eq(t2[1].get('inet:udp6:port'), 5000)
self.eq(t2[1].get('inet:udp6:ipv6'), '::3:2:1')
self.eq(core.getTypeRepr('inet:tcp6', '[0:0:0:0:0:0:0:1]:80'),
'[::1]:80')
self.eq(core.getTypeRepr('inet:tcp6', '[::1]:80'), '[::1]:80')
self.eq(core.getTypeRepr('inet:tcp6', '[0:0:0:0:0:3:2:1]:5000'),
'[::3:2:1]:5000')
def test_model_file_seeds_capitalization(self):
fhash = '6ACC29BFC5F8F772FA7AAF4A705F91CB68DC88CB22F4EF5101281DC42109A104'
fhash_lower = fhash.lower()
stable_guid = 'ed73917b1dc4011627f7a101ace491c8'
with self.getRamCore() as core:
n1 = core.formTufoByProp('file:bytes:sha256', fhash)
n2 = core.formTufoByProp('file:bytes:sha256', fhash_lower)
# Sha256 should be lowercase since the prop type is lowercased
n1def = s_tufo.ndef(n1)
n2def = s_tufo.ndef(n2)
self.eq(n1def[1], stable_guid)
self.eq(n2def[1], stable_guid)
self.eq(n1[0], n2[0])
def test_model_ibutton(self):
td = {'serial': '5E00000038013B21', 'block': '3', 'tmnt': 'Control'}
ed = {'serial': '5e00000038013b21', 'block': 3, 'tmnt': 'control'}
eprop = 'a14c37660532af1e02e91a2c0cda75f0'
with self.getRamCore() as core: # s_cortex_common.Cortex
node = core.formTufoByProp('ibutton', td)
_, pprop = s_tufo.ndef(node)
props = s_tufo.props(node)
self.eq(pprop, eprop)
self.eq(props, ed)
with self.getRamCore() as core: # s_cortex_common.Cortex
new_td = td.copy()
new_td['serial'] = new_td.get('serial').lower()
node = core.formTufoByProp('ibutton', new_td)
_, pprop = s_tufo.ndef(node)
props = s_tufo.props(node)
self.eq(pprop, eprop)
self.eq(props, ed)
def test_storm_refs(self):
with self.getRamCore() as core:
core.formTufoByProp('inet:dns:a', 'foo.com/1.2.3.4')
core.formTufoByProp('inet:dns:a', 'bar.com/1.2.3.4')
self.eq(len(core.eval('inet:ipv4=1.2.3.4 refs(in)')), 3)
self.eq(len(core.eval('inet:ipv4=1.2.3.4 refs(in,limit=1)')), 2)
self.eq(len(core.eval('inet:dns:a=foo.com/1.2.3.4 refs(out)')), 3)
self.eq(
len(core.eval('inet:dns:a=foo.com/1.2.3.4 refs(out,limit=1)')),
2)
# Try refs() with a file:txtref node. This uses propvalu to do the pivoting
fnode = core.formTufoByProp('file:bytes:md5',
'd41d8cd98f00b204e9800998ecf8427e')
form, pprop = s_tufo.ndef(fnode)
node = core.formTufoByProp('file:txtref',
'({},inet:ipv4=1.2.3.4)'.format(pprop))
nodes = core.eval('file:txtref refs()')
self.eq(len(nodes), 3)
forms = {s_tufo.ndef(node)[0] for node in nodes}
self.eq(forms, {'inet:ipv4', 'file:bytes', 'file:txtref'})
# Make sure we're also pivoting on something with a "=" in the valu
# since we have to do a split operation inside of the refs() operator
self.nn(core.formTufoByProp('inet:passwd', 'oh=my=graph!'))
node = core.formTufoByProp(
'file:txtref', '({},"inet:passwd=oh=my=graph!")'.format(pprop))
form, pprop = s_tufo.ndef(node)
nodes = core.eval('file:txtref={} refs()'.format(pprop))
self.eq(len(nodes), 3)
forms = {s_tufo.ndef(node)[0] for node in nodes}
self.eq(forms, {'file:bytes', 'file:txtref', 'inet:passwd'})
# Try refs() with a non-XREF type which has propvalu properties.
f1 = core.formTufoByProp('pvsub', 'hai', xref='inet:ipv4=1.2.3.4')
self.nn(f1)
nodes = core.eval('pvsub refs()')
self.eq(len(nodes), 2)
def test_model_orgs_oumember(self):
with self.getRamCore() as core:
pnode = core.formTufoByProp('ps:person', '*', name='grey, robert')
_, pprop = s_tufo.ndef(pnode)
onode = core.formTufoByProp('ou:org:name', 'derry sanitation corp')
_, oprop = s_tufo.ndef(onode)
mnode = core.formTufoByProp(
'ou:member', {
'org': oprop,
'person': pprop
}, **{
'start': '2017',
'end': '2018',
'title': 'Dancing Clown'
})
self.nn(mnode)
_, mpprop = s_tufo.ndef(mnode)
props = s_tufo.props(mnode)
self.eq(props.get('org'), oprop)
self.eq(props.get('person'), pprop)
self.eq(props.get('end'), core.getTypeNorm('time', '2018')[0])
self.eq(props.get('start'), core.getTypeNorm('time', '2017')[0])
self.eq(props.get('title'), 'dancing clown')
# We can traverse across the ou:member node
nodes = core.eval(
'ps:person=%s -> ou:member:person :org -> ou:org' % pprop)
self.len(1, nodes)
self.eq(oprop, nodes[0][1].get('ou:org'))
nodes = core.eval(
'ou:org=%s -> ou:member:org :person -> ps:person' % oprop)
self.len(1, nodes)
self.eq(pprop, nodes[0][1].get('ps:person'))
def main(options): # pragma: no cover
if not options.verbose:
logging.disable(logging.DEBUG)
from pprint import pprint
order_d = parse_order_csv(options.order)
with s_cortex.openurl(options.core) as core:
core.setConfOpts({
'modules': (('ibutton.model.IButtonModel', {}), ),
'caching': 1,
'cache:maxsize': 25000
})
with core.getCoreXact() as xact:
fps = [options.input]
if os.path.isdir(options.input):
fps = [
os.path.join(options.input, fn)
for fn in os.listdir(options.input)
]
for fp in fps:
log.info('Processing %s', fp)
for i, button in enumerate(parse_ibutton_csv(fp, order_d)):
log.debug('Button # %s', i)
button.check_integrity()
# Make ibutton node
d = {
SERIAL: button.metadata.get('Logger serial number'),
BLOCK: button.metadata.get(BLOCK),
TMNT: button.metadata.get(TMNT),
}
bnode = core.formTufoByProp('ibutton', d)
_, pprop = s_tufo.ndef(bnode)
if bnode[1].get('ibutton:serial') == "":
log.warning('Missing serial for ibutton!')
for k, v in button.rows:
# XXX TODO - timezone correction???
rtime = k.strftime(SYN_TIME_FORMAT)
d = {'button': pprop, 'rtime': rtime}
node = core.formTufoByProp('idata', d, temp=v)
if not node[1].get('.new'):
pass
# print('node existed!')
# print(node)
sys.exit(0)
def analyze(core, fdir):
d = {}
for inode in core.eval('ibutton'):
_, pprop = s_tufo.ndef(inode)
log.info('Collecting data for {}'.format(pprop))
ibutton = button.ButtonData()
props = s_tufo.props(inode)
ibutton.metadata[TMNT] = props.get(TMNT)
ibutton.metadata[BLOCK] = props.get(BLOCK)
for node in core.eval('idata:button=%s' % pprop):
props = s_tufo.props(node)
tick = props.get('rtime')
# Taken from synapse.lib.time
dt = datetime.datetime(1970, 1,
1) + datetime.timedelta(milliseconds=tick)
# from dC (int) to C (float)
v = float(props.get('temp')) / 10
ibutton.append((dt, v))
d[pprop] = ibutton
summer_rows = []
for pprop, ibutton in d.items():
log.info('Analyzing data for {}'.format(pprop))
ibutton.analyze_button()
ibutton.write_computed_data_to_files(fdir)
for k, v in ibutton.summer_gdd.items():
d = {
'year': k,
'gdd': v,
TMNT: ibutton.metadata.get(TMNT),
BLOCK: ibutton.metadata.get(BLOCK)
}
summer_rows.append(d)
fn = 'aggregate_summer_gdd.csv'
fp = os.path.join(fdir, fn)
with open(fp, 'w') as f:
w = csv.DictWriter(f, [TMNT, BLOCK, 'year', 'gdd'])
w.writeheader()
for d in summer_rows:
w.writerow(d)
total_summer_gdd = sum([d.get('gdd') for d in summer_rows])
log.info('Total summer gdd rows: [%s] [%s]', total_summer_gdd,
(total_summer_gdd / len(summer_rows)))
def test_model_infotech_software(self):
with self.getRamCore() as core:
a1 = core.formTufoByProp('inet:web:acct', 'vertex.link/pennywise')
o1 = core.formTufoByProp('ou:org:alias', 'deadlights')
p1 = core.formTufoByProp('ps:person',
[['guidname', 'Robert Gray']])
h1 = core.formTufoByProp('it:host', '(name=pennywise001)')
self.nn(a1)
self.nn(o1)
self.nn(p1)
self.nn(h1)
# Random guid
s1 = core.formTufoByProp(
'it:prod:soft', '*', **{
'name': 'Balloon Maker',
'desc': "Pennywise's patented balloon blower uper",
'desc:short': 'Balloon blower',
'author:org': o1[1].get('ou:org'),
'author:acct': a1[1].get('inet:web:acct'),
'author:person': p1[1].get('ps:person'),
'url': 'https://vertex.link/products/balloonmaker'
})
self.eq(s1[1].get('it:prod:soft:name'), 'balloon maker')
self.eq(s1[1].get('it:prod:soft:desc'),
"Pennywise's patented balloon blower uper")
self.eq(s1[1].get('it:prod:soft:desc:short'), 'balloon blower')
self.eq(
s1[1].get('it:prod:soft:author:acct'),
a1[1].get('inet:web:acct'),
)
self.eq(s1[1].get('it:prod:soft:author:org'), o1[1].get('ou:org'))
self.eq(s1[1].get('it:prod:soft:author:person'),
p1[1].get('ps:person'))
self.eq(s1[1].get('it:prod:soft:url'),
'https://vertex.link/products/balloonmaker')
# Stable guid
s2 = core.formTufoByProp(
'it:prod:soft',
'(name="Balloon Maker",author:acct=vertex.link/pennywise)')
self.eq(s2[1].get('it:prod:soft'),
'5fd0340d2ad8878fe53ccd28843ff2dc')
self.raises(PropNotFound, core.formTufoByProp, 'it:prod:soft', '*')
nodes = core.getTufosByProp('it:prod:soft:name', 'balloon maker')
self.len(2, nodes)
# Make some version nodes
# Ensure :semver is populated by the vers value.
sv1 = core.formTufoByProp(
'it:prod:softver',
'*',
software='5fd0340d2ad8878fe53ccd28843ff2dc',
vers='V1.0.1',
url='https://vertex.link/products/balloonmaker/release_101.exe'
)
self.eq(sv1[1].get('it:prod:softver:software'),
'5fd0340d2ad8878fe53ccd28843ff2dc')
self.eq(sv1[1].get('it:prod:softver:software:name'),
'balloon maker')
self.eq(sv1[1].get('it:prod:softver:vers'), 'V1.0.1')
self.eq(sv1[1].get('it:prod:softver:vers:norm'), 'v1.0.1')
self.eq(sv1[1].get('it:prod:softver:semver'), 0x000010000000001)
self.eq(sv1[1].get('it:prod:softver:semver:major'), 1)
self.eq(sv1[1].get('it:prod:softver:semver:minor'), 0)
self.eq(sv1[1].get('it:prod:softver:semver:patch'), 1)
self.eq(
sv1[1].get('it:prod:softver:url'),
'https://vertex.link/products/balloonmaker/release_101.exe')
# Link the softver to a host
hs1 = core.formTufoByProp(
'it:hostsoft',
(h1[1].get('it:host'), sv1[1].get('it:prod:softver')), **{
'seen:min': '2013',
'seen:max': '2017',
})
self.eq(hs1[1].get('it:hostsoft:host'), h1[1].get('it:host'))
self.eq(hs1[1].get('it:hostsoft:softver'),
sv1[1].get('it:prod:softver'))
self.eq(hs1[1].get('it:hostsoft:seen:min'),
core.getTypeNorm('time', '2013')[0])
self.eq(hs1[1].get('it:hostsoft:seen:max'),
core.getTypeNorm('time', '2017')[0])
nodes = core.eval(
'it:prod:soft:name="balloon maker" ->it:prod:softver:software ->it:hostsoft:softver '
':host->it:host')
self.len(1, nodes)
form, prop = s_tufo.ndef(nodes[0])
self.eq(form, 'it:host')
self.eq(prop, 'e862b0d7f3a9e015171a4113e0dbe861')
# Go backwards from the host
nodes = core.eval(
'it:host=e862b0d7f3a9e015171a4113e0dbe861 ->it:hostsoft:host :softver '
'->it:prod:softver')
self.len(1, nodes)
form, prop = s_tufo.ndef(nodes[0])
self.eq(form, 'it:prod:softver')
self.eq(prop, sv1[1].get('it:prod:softver'))
# Make a bunch of softver nodes in order to do filtering via storm
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'V1.0.0')))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'V1.1.0')))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'V0.0.1')))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'V0.1.0')))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', '0.1.1')))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'V2.0.0-alpha+b1')))
nodes = core.eval('it:prod:softver:semver<1.0.0')
self.len(3, nodes)
nodes = core.eval(
'it:prod:softver:semver<1.0.0 +it:prod:softver:semver:minor=1')
self.len(2, nodes)
nodes = core.eval('it:prod:softver:semver<=1.0.0')
self.len(4, nodes)
nodes = core.eval('it:prod:softver:semver>1.1.0')
self.len(1, nodes)
nodes = core.eval('it:prod:softver:semver:pre=alpha')
self.len(1, nodes)
# Try some non-standard semver values
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', ' OhMy2016-12-10 ')))
self.eq(sv[1].get('it:prod:softver:vers'), ' OhMy2016-12-10 ')
self.eq(sv[1].get('it:prod:softver:vers:norm'), 'ohmy2016-12-10')
self.eq(sv[1].get('it:prod:softver:semver:major'), 2016)
self.eq(sv[1].get('it:prod:softver:semver:minor'), 12)
self.eq(sv[1].get('it:prod:softver:semver:patch'), 10)
nodes = core.eval('it:prod:softver:semver:major=2016')
self.len(1, nodes)
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', '1.2')))
self.eq(sv[1].get('it:prod:softver:semver:major'), 1)
self.eq(sv[1].get('it:prod:softver:semver:minor'), 2)
self.none(sv[1].get('it:prod:softver:semver:patch'))
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', '1.2.3.4')))
self.eq(sv[1].get('it:prod:softver:semver:major'), 1)
self.eq(sv[1].get('it:prod:softver:semver:minor'), 2)
self.eq(sv[1].get('it:prod:softver:semver:patch'), 3)
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'AlPHa')))
self.eq(sv[1].get('it:prod:softver:vers'), 'AlPHa')
self.eq(sv[1].get('it:prod:softver:vers:norm'), 'alpha')
self.none(sv[1].get('it:prod:softver:semver'))
# Set the software:name directly
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'beta')), **{'software:name': 'ballon maker-beta'})
self.eq(sv[1].get('it:prod:softver:vers'), 'beta')
self.eq(sv[1].get('it:prod:softver:software:name'),
'ballon maker-beta')
# Set the semver value separately from the vers.
# This bypasses the node:form callback from setting the prop and instead
# relys on regular prop-norming.
sv = core.formTufoByProp(
'it:prod:softver',
(('software', '5fd0340d2ad8878fe53ccd28843ff2dc'),
('vers', 'AlPHa001')),
semver='0.0.1-alpha+build.001')
self.eq(sv[1].get('it:prod:softver:vers'), 'AlPHa001')
self.eq(sv[1].get('it:prod:softver:vers:norm'), 'alpha001')
self.eq(sv[1].get('it:prod:softver:semver'), 1)
self.eq(sv[1].get('it:prod:softver:semver:major'), 0)
self.eq(sv[1].get('it:prod:softver:semver:minor'), 0)
self.eq(sv[1].get('it:prod:softver:semver:patch'), 1)
self.eq(sv[1].get('it:prod:softver:semver:build'), 'build.001')
self.eq(sv[1].get('it:prod:softver:semver:pre'), 'alpha')
def _stormOperTree(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if not args:
raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])')
limt = self.getLiftLimitHelp(opts.get('limit'))
core = self.getStormCore()
# Prevent infinite pivots
try:
recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20))
except s_common.BadTypeValu as e:
raise s_common.BadOperArg(oper='tree', name='recurlim', mesg=e.errinfo.get('mesg'))
if recurlim < 0:
raise s_common.BadOperArg(oper='tree', name='recurlim', mesg='must be >= 0')
srcp = None
dstp = args[0]
if len(args) > 1:
srcp = args[0]
dstp = args[1]
# do we have a relative source property?
relsrc = srcp is not None and srcp.startswith(':')
tufs = query.data()
queried_vals = set()
while True:
vals = set()
if srcp is not None and not relsrc:
for tufo in tufs:
valu = tufo[1].get(srcp)
if valu is not None:
vals.add(valu)
elif not relsrc:
for tufo in tufs:
form, valu = s_tufo.ndef(tufo)
if valu is not None:
vals.add(valu)
else:
for tufo in tufs:
form, _ = s_tufo.ndef(tufo)
valu = tufo[1].get(form + srcp)
if valu is not None:
vals.add(valu)
qvals = list(vals - queried_vals)
if not qvals:
break
nodes = core.stormTufosBy('in', dstp, qvals, limit=limt.get())
[query.add(n) for n in nodes]
if limt.dec(len(nodes)):
break
queried_vals = queried_vals.union(vals)
if recurlim > 0:
recurlim -= 1
if recurlim < 1:
break
tufs = query.data()
def _stormOperTree(self, query, oper):
args = oper[1].get('args')
opts = dict(oper[1].get('kwlist'))
if not args:
raise s_common.BadSyntaxError(mesg='tree([<srcprop>], <destprop>, [recurlim=<limit>])')
core = self.getStormCore()
# Prevent infinite pivots
recurlim, _ = core.getTypeNorm('int', opts.get('recurlim', 20))
srcp = None
dstp = args[0]
if len(args) > 1:
srcp = args[0]
dstp = args[1]
# do we have a relative source property?
relsrc = srcp is not None and srcp.startswith(':')
tufs = query.data()
queried_vals = set()
while True:
vals = set()
if srcp is not None and not relsrc:
for tufo in tufs:
valu = tufo[1].get(srcp)
if valu is not None:
vals.add(valu)
elif not relsrc:
for tufo in tufs:
form, valu = s_tufo.ndef(tufo)
if valu is not None:
vals.add(valu)
else:
for tufo in tufs:
form, _ = s_tufo.ndef(tufo)
valu = tufo[1].get(form + srcp)
if valu is not None:
vals.add(valu)
qvals = list(vals - queried_vals)
if not qvals:
break
[query.add(t) for t in self.stormTufosBy('in', dstp, qvals, limit=opts.get('limit'))]
queried_vals = queried_vals.union(vals)
if recurlim:
recurlim -= 1
if recurlim < 1:
break
tufs = query.data()
def test_axon_host_clone(self):
self.skipLongTest()
self.thisHostMustNot(platform='windows')
busurl = 'local://%s/axons' % guid()
dmon = s_daemon.Daemon()
dmon.listen(busurl)
dmon.share('axons', s_service.SvcBus(), fini=True)
with self.getTestDir() as datadir:
dir0 = gendir(datadir, 'host0')
dir1 = gendir(datadir, 'host1')
dir2 = gendir(datadir, 'host2')
host0 = s_axon.AxonHost(dir0, **{'axon:hostname': 'host0',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host1 = s_axon.AxonHost(dir1, **{'axon:hostname': 'host1',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
host2 = s_axon.AxonHost(dir2, **{'axon:hostname': 'host2',
'axon:axonbus': busurl,
'axon:bytemax': s_axon.megabyte * 100,
})
props = {
'axon:bytemax': s_axon.megabyte * 50,
}
axfo0 = host0.add(**props)
axon0 = s_telepath.openlink(axfo0[1].get('link')) # type: s_axon.Axon
self.true(axon0._waitClonesReady(timeout=16))
self.notin(axfo0[0], host0.cloneaxons)
# get refs to axon0's clones
ciden1 = host1.cloneaxons[0]
axonc1 = host1.axons.get(ciden1) # type: s_axon.Axon
self.eq(axonc1.getConfOpt('axon:clone:iden'), axfo0[0])
ciden2 = host2.cloneaxons[0]
axonc2 = host2.axons.get(ciden2) # type: s_axon.Axon
self.eq(axonc2.getConfOpt('axon:clone:iden'), axfo0[0])
# Ensure axon:clone events have fired on the cores
iden = axon0.alloc(100)
cv = b'V' * 100
blob = axon0.chunk(iden, cv)
# We need to check the clones of axon0 to ensure that the clones have the data too
time.sleep(1)
self.nn(blob)
self.true(axon0.has('md5', blob[1].get('axon:blob:md5')))
self.true(axon0.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axon0.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axon0.iterblob(blob))
self.eq(axonbyts, cv)
self.true(axonc1.has('md5', blob[1].get('axon:blob:md5')))
self.true(axonc1.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axonc1.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axonc1.iterblob(blob))
self.eq(axonbyts, cv)
self.true(axonc2.has('md5', blob[1].get('axon:blob:md5')))
self.true(axonc2.has('sha1', blob[1].get('axon:blob:sha1')))
self.true(axonc2.has('sha256', blob[1].get('axon:blob:sha256')))
axonbyts = b''.join(_byts for _byts in axonc2.iterblob(blob))
self.eq(axonbyts, cv)
# Now write a large amount of data to axon0 and ensure that data is replicated
blobs = []
n = 5
tststr = 'deadb33f' * 100000
for i in range(n):
byts = tststr + 'lol' * i
byts = byts.encode()
blob = axon0.eatbytes(byts)
blobs.append(blob)
self.eq(len(blobs), n)
time.sleep(2)
for i, blob in enumerate(blobs):
form, pprop = s_tufo.ndef(blob)
ret = axonc1.byiden(pprop)
self.nn(ret)
ret = axonc2.byiden(pprop)
self.nn(ret)
# Add axons to the other two hosts and see that they clone over to the other hosts
axfo1 = host1.add(**props)
axon1 = s_telepath.openlink(axfo1[1].get('link')) # type: s_axon.Axon
self.true(axon1._waitClonesReady(timeout=16))
axfo2 = host2.add(**props)
axon2 = s_telepath.openlink(axfo2[1].get('link')) # type: s_axon.Axon
self.true(axon2._waitClonesReady(timeout=16))
self.eq(len(host0.axons), 3)
self.eq(len(host1.axons), 3)
self.eq(len(host2.axons), 3)
self.eq(len(host0.cloneaxons), 2)
self.eq(len(host1.cloneaxons), 2)
self.eq(len(host2.cloneaxons), 2)
# Fini the proxy objects
axonc1.fini()
axonc2.fini()
axon0.fini()
axon1.fini()
axon2.fini()
# Fini the hosts
host0.fini()
host1.fini()
host2.fini()
# Ensure the axonhost fini'd its objects
self.true(host0.axonbus.isfini)
for axon in host0.axons.values():
self.true(axon.isfini)
dmon.fini()
