def test_model_dns_req(self):
with self.getRamCore() as core:
now = s_common.now()
node = core.formTufoByProp('inet:dns:req',
('1.2.3.4', 'VERTEX.link', 'A'), **{
'seen:min': now,
'seen:max': now
})
self.eq(node[1].get('inet:dns:req:type'), 'a')
self.eq(node[1].get('inet:dns:req:client'), 'tcp://1.2.3.4')
self.eq(node[1].get('inet:dns:req:client:ipv4'), 0x01020304)
self.eq(node[1].get('inet:dns:req:fqdn'), 'vertex.link')
self.eq(node[1].get('inet:dns:req:seen:min'), now)
self.eq(node[1].get('inet:dns:req:seen:max'), now)
newnow = s_common.now() + 100
node = core.setTufoProps(
node, **{
'seen:min': newnow,
'seen:max': newnow
})
self.eq(node[1].get('inet:dns:req:seen:min'), now)
self.eq(node[1].get('inet:dns:req:seen:max'), newnow)
def save(self, items):
'''
Save a series of items to a sequence.
Args:
items (tuple): The series of items to save into the sequence.
Returns:
The index of the first item
'''
rows = []
indx = self.indx
size = 0
tick = s_common.now()
for item in items:
byts = s_msgpack.en(item)
size += len(byts)
lkey = s_common.int64en(indx)
indx += 1
rows.append((lkey, byts))
self.slab.putmulti(rows, append=True, db=self.db)
took = s_common.now() - tick
origindx = self.indx
self.indx = indx
return {'indx': indx, 'size': size, 'count': len(items), 'time': tick, 'took': took, 'orig': origindx}
def convert_rstorm(_):
import subprocess
import synapse
import synapse.common as s_common
abssynf = os.path.abspath(synapse.__file__)
synbd = os.path.split(abssynf)[0] # Split off __init__
synpd = os.path.split(synbd)[0] # split off the synapse module directory
env = {**os.environ, 'SYN_LOG_LEVEL': 'DEBUG'}
cwd = os.getcwd()
for fdir, dirs, fns in os.walk(cwd):
for fn in fns:
if fn.endswith('.rstorm'):
oname = fn.rsplit('.', 1)[0]
oname = oname + '.rst'
sfile = os.path.join(fdir, fn)
ofile = os.path.join(fdir, oname)
tick = s_common.now()
args = ['python', '-m', 'synapse.tools.rstorm', '--save', ofile, sfile]
r = subprocess.run(args, cwd=synpd, env=env)
assert r.returncode == 0, f'Failed to convert {sfile}'
tock = s_common.now()
took = (tock - tick) / 1000
print(f'convert_rstorm: Rstorm {fn} execution took {took} seconds.')
def _normPyStr(self, valu):
valu = valu.strip().lower()
if valu == 'now':
return self._normPyInt(s_common.now())
# an unspecififed time in the future...
if valu == '?':
return self.futsize, {}
# self contained relative time string
# we need to be pretty sure this is meant for us, otherwise it might
# just be a slightly messy time parse
unitcheck = [u for u in s_time.timeunits.keys() if u in valu]
if unitcheck and '-' in valu or '+' in valu:
splitter = '+'
if '-' in valu:
splitter = '-'
bgn, end = valu.split(splitter, 1)
delt = s_time.delta(splitter + end)
if bgn:
bgn = self._normPyStr(bgn)[0]
else:
bgn = s_common.now()
return self._normPyInt(delt + bgn)
valu = s_time.parse(valu)
return self._normPyInt(valu)
async def runStorm():
cancelled = False
tick = s_common.now()
count = 0
try:
# Always start with an init message.
await chan.put(('init', {'tick': tick, 'text': text, 'task': synt.iden}))
# Try text parsing. If this fails, we won't be able to get a storm
# runtime in the snap, so catch and pass the `err` message
await self.core.getStormQuery(text, mode=mode)
shownode = (not show or 'node' in show)
async with await self.snap(user=user) as snap:
if not show:
snap.link(chan.put)
else:
[snap.on(n, chan.put) for n in show]
if shownode:
async for pode in snap.iterStormPodes(text, opts=opts, user=user):
await chan.put(('node', pode))
count += 1
else:
async for item in snap.storm(text, opts=opts, user=user):
count += 1
except s_stormctrl.StormExit:
pass
except asyncio.CancelledError:
logger.warning('Storm runtime cancelled.',
extra={'synapse': {'text': text, 'username': user.name, 'user': user.iden}})
cancelled = True
raise
except Exception as e:
logger.exception(f'Error during storm execution for {{ {text} }}',
extra={'synapse': {'text': text, 'username': user.name, 'user': user.iden}})
enfo = s_common.err(e)
enfo[1].pop('esrc', None)
enfo[1].pop('ename', None)
await chan.put(('err', enfo))
finally:
if not cancelled:
tock = s_common.now()
took = tock - tick
await chan.put(('fini', {'tock': tock, 'took': took, 'count': count}))
def puts(self, items, seqn=None):
'''
Add the structured data from items to the CryoTank.
Args:
items (list): A list of objects to store in the CryoTank.
seqn (iden, offs): An iden / offset pair to record.
Returns:
int: The ending offset of the items or seqn.
'''
itembyts = [s_msgpack.en(i) for i in items]
tick = s_common.now()
bytesize = sum([len(b) for b in itembyts])
with self.lenv.begin(db=self.lenv_items, write=True) as xact:
todo = []
for byts in itembyts:
todo.append((struct.pack('>Q', self.items_indx), byts))
self.items_indx += 1
retn = self.items_indx
with xact.cursor() as curs:
curs.putmulti(todo, append=True)
took = s_common.now() - tick
with xact.cursor(db=self.lenv_metrics) as curs:
lkey = struct.pack('>Q', self.metrics_indx)
self.metrics_indx += 1
info = {
'time': tick,
'count': len(items),
'size': bytesize,
'took': took
}
curs.put(lkey, s_msgpack.en(info), append=True)
if seqn is not None:
iden, offset = seqn
nextoff = offset + len(items)
self.offs.xset(xact, iden, nextoff)
retn = nextoff
self.schedCoroSafe(self.fire('cryotank:puts',
numrecords=len(itembyts)))
return retn
async def _initTestRunts(self):
modl = self.core.model
fnme = 'test:runt'
form = modl.form(fnme)
now = s_common.now()
data = [
(' BEEP ', {
'tick': modl.type('time').norm('2001')[0],
'lulz': 'beep.sys',
'.created': now
}),
('boop', {
'tick': modl.type('time').norm('2010')[0],
'.created': now
}),
('blah', {
'tick': modl.type('time').norm('2010')[0],
'lulz': 'blah.sys'
}),
('woah', {}),
]
for pprop, propd in data:
props = {}
pnorm, _ = form.type.norm(pprop)
for k, v in propd.items():
prop = form.props.get(k)
if prop:
norm, _ = prop.type.norm(v)
props[k] = norm
props.setdefault('.created', s_common.now())
rows = [('*' + fnme, pnorm)]
for k, v in props.items():
rows.append((k, v))
buid = s_common.buid((fnme, pnorm))
self._runtsByBuid[buid] = rows
# Allow for indirect lookup to a set of buids
self._runtsByPropValu[fnme].append(buid)
self._runtsByPropValu[(fnme, pnorm)].append(buid)
for k, propvalu in props.items():
prop = fnme + ':' + k
if k.startswith('.'):
prop = fnme + k
self._runtsByPropValu[prop].append(buid)
if modl.prop(prop).type.indx(propvalu):
# Can the secondary property be indexed for lift?
self._runtsByPropValu[(prop, propvalu)].append(buid)
def backup(srcdir, dstdir, compact=True):
'''
Args:
compact (bool): whether to optimize storage while copying to the destination
'''
tick = s_common.now()
srcdir = s_common.reqdir(srcdir)
dstdir = s_common.gendir(dstdir)
logger.info(f'Starting backup of [{srcdir}]')
logger.info(f'Destination dir: [{dstdir}]')
for root, dnames, fnames in os.walk(srcdir, topdown=True):
relpath = os.path.relpath(root, start=srcdir)
for name in list(dnames):
# Explicitly skip directory names of 'tmp' to avoid backing up temporary files
if name == 'tmp':
dnames.remove(name)
continue
srcpath = s_common.genpath(root, name)
dstpath = s_common.genpath(dstdir, relpath, name)
if name.endswith('.lmdb'):
dnames.remove(name)
backup_lmdb(srcpath, dstpath)
continue
logger.info(f'making dir:{dstpath}')
s_common.gendir(dstpath)
for name in fnames:
srcpath = s_common.genpath(root, name)
# skip unix sockets etc...
if not os.path.isfile(srcpath):
continue
dstpath = s_common.genpath(dstdir, relpath, name)
logger.info(f'copying: {srcpath} -> {dstpath}')
shutil.copy(srcpath, dstpath)
tock = s_common.now()
logger.info(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
return
def openLogFd(self, opts):
opath = self.locs.get('log:fp')
if opath:
self.printf('Must call --off to disable current file before starting a new file.')
return
fmt = opts.format
path = opts.path
nodes_only = opts.nodes_only
splice_only = opts.splices_only
if not path:
ts = s_time.repr(s_common.now(), True)
fn = f'storm_{ts}.{fmt}'
path = s_common.getSynPath('stormlogs', fn)
self.printf(f'Starting logfile at [{path}]')
q = queue.Queue()
fd = s_common.genfile(path)
# Seek to the end of the file. Allows a user to append to a file.
fd.seek(0, 2)
self.locs['log:fp'] = path
self.locs['log:fd'] = fd
self.locs['log:fmt'] = fmt
self.locs['log:queue'] = q
self.locs['log:thr'] = self.queueLoop()
self.locs['log:nodesonly'] = nodes_only
self.locs['log:splicesonly'] = splice_only
self._cmd_cli.on('storm:mesg', self.onStormMesg)
async def _initTriggerRunts(self):
now = s_common.now()
typeform = self.model.form('syn:trigger')
for iden, trig in await self.core.listTriggers():
tnorm, _ = typeform.type.norm(iden)
props = {'.created': now,
'doc': trig.doc,
'name': trig.name,
'vers': trig.ver,
'cond': trig.cond,
'storm': trig.storm,
'enabled': trig.enabled,
'user': self.core.getUserName(trig.useriden),
}
if trig.tag is not None:
props['tag'] = trig.tag
if trig.form is not None:
props['form'] = trig.form
if trig.prop is not None:
props['prop'] = trig.prop
self._addRuntRows('syn:trigger', tnorm, props,
self._triggerRuntsByBuid, self._triggerRuntsByPropValu)
async def _setPriority(self, valu):
self.proj.confirm(('project', 'ticket', 'set', 'priority'))
strvalu = await tostr(valu)
await self.node.set('priority', strvalu)
await self.node.set('updated', s_common.now())
def _revModl201706201837(self):
'''
Add :port and :ipv4 to inet:tcp4 and inet:udp4 nodes.
'''
tick = s_common.now()
forms = ('inet:tcp4', 'inet:udp4')
for form in forms:
adds = []
portprop = '{}:port'.format(form)
ipv4prop = '{}:ipv4'.format(form)
rows = self.core.getRowsByProp(form)
for i, p, v, _ in rows:
norm, subs = s_datamodel.tlib.getTypeNorm(form, v)
port = subs.get('port')
if port:
adds.append((i, portprop, port, tick))
ipv4 = subs.get('ipv4')
if ipv4:
adds.append((i, ipv4prop, ipv4, tick))
if adds:
self.core.addRows(adds)
async def test_syningest_remote(self):
async with self.getTestCore() as core:
guid = s_common.guid()
seen = s_common.now()
gestdef = self.getIngestDef(guid, seen)
with self.getTestDir() as dirn:
# Test yaml support here
gestfp = s_common.genpath(dirn, 'gest.yaml')
s_common.yamlsave(gestdef, gestfp)
argv = ['--cortex', core.getLocalUrl(),
'--debug',
'--modules', 'synapse.tests.utils.TestModule',
gestfp]
outp = self.getTestOutp()
cmdg = s_t_utils.CmdGenerator(['storm test:pivcomp -> *', EOFError()])
with self.withCliPromptMockExtendOutp(outp):
with self.withTestCmdr(cmdg):
self.eq(await s_feed.main(argv, outp=outp), 0)
self.true(outp.expect('test:str=haha', throw=False))
self.true(outp.expect('test:pivtarg=hehe', throw=False))
def openLogFd(self, opts):
opath = self.locs.get('log:fp')
if opath:
self.printf(
'Must call --off to disable current file before starting a new file.'
)
return
fmt = opts.format
path = opts.path
nodes_only = opts.nodes_only
edits_only = opts.edits_only
if not path:
ts = s_time.repr(s_common.now(), True)
fn = f'storm_{ts}.{fmt}'
path = s_common.getSynPath('stormlogs', fn)
self.printf(f'Starting logfile at [{path}]')
q = queue.Queue()
fd = s_common.genfile(path)
# Seek to the end of the file. Allows a user to append to a file.
fd.seek(0, 2)
self.locs['log:fp'] = path
self.locs['log:fd'] = fd
self.locs['log:fmt'] = fmt
self.locs['log:queue'] = q
self.locs['log:thr'] = self.queueLoop()
self.locs['log:nodesonly'] = nodes_only
self.locs['log:editsonly'] = edits_only
self._cmd_cli.on('storm:mesg', self.onStormMesg)
async def _funcMacroSet(self, name, storm):
'''
Add or modify an existing Storm Macro in the Cortex.
Args:
name (str): Name of the Storm Macro to add or modify.
storm (str): The Storm query to add to the macro.
Returns:
None: Returns None.
'''
name = await s_stormtypes.tostr(name)
storm = await s_stormtypes.tostr(storm)
# validation
await self.runt.getStormQuery(storm)
path = ('cortex', 'storm', 'macros', name)
user = self.runt.user
mdef = await self.runt.snap.core.getHiveKey(path)
if mdef is not None:
if mdef['user'] != user.iden and not user.isAdmin():
mesg = 'Macro belongs to a different user'
raise s_exc.AuthDeny(mesg=mesg)
mdef = {
'user': user.iden,
'storm': storm,
'edited': s_common.now(),
}
await self.runt.snap.core.setHiveKey(path, mdef)
async def _srcPullLyrSplices(self, lyriden):
'''
Open a proxy to the source layer and initiates splice reader.
Intended to be run as a free-running task, and will poll for updates every poll_s.
Args:
lyriden (str): Layer iden
'''
trycnt = 0
q_cap = self.q_cap
poll_s = self.poll_s
turl = self._getLayerUrl(self.src, lyriden)
while not self.isfini:
try:
trycnt += 1
prx = await s_telepath.openurl(turl)
trycnt = 0
logger.info(f'Connected to source {lyriden}')
islive = False
while not prx.isfini:
queue = self._queues.get(lyriden)
startoffs = self.pull_offs.get(lyriden)
logger.debug(f'Pulling splices for layer {lyriden} starting from offset {startoffs}')
if islive:
self._pull_status[lyriden] = 'reading_at_live'
else:
self._pull_status[lyriden] = 'reading_catchup'
self._pull_evnts[lyriden].clear()
self.pull_last_start[lyriden] = s_common.now()
nextoffs, islive = await self._srcIterLyrSplices(prx, startoffs, queue)
self.pull_offs.set(lyriden, nextoffs)
while not islive and len(queue.linklist) > q_cap:
await asyncio.sleep(1)
if queue.isfini:
logger.warning(f'Queue is finid; stopping {lyriden} src pull')
self._pull_status[lyriden] = 'queue_fini'
return
if islive:
if nextoffs == startoffs:
logger.debug(f'All splices from {lyriden} have been read')
self._pull_status[lyriden] = 'up_to_date'
self._pull_evnts[lyriden].set()
await asyncio.sleep(poll_s)
except asyncio.CancelledError: # pragma: no cover
raise
except (ConnectionError, s_exc.IsFini):
logger.exception(f'Source layer connection error cnt={trycnt}: {lyriden}')
self._pull_status[lyriden] = 'connect_err'
await asyncio.sleep(2 ** trycnt)
def _revModl201711012123(self):
now = s_common.now()
forms = sorted(self.core.getTufoForms())
nforms = len(forms)
for n, form in enumerate(forms):
adds = []
logger.debug('Computing node:ndef rows for [{}]'.format(form))
for i, p, v, t in self.core.store.getRowsByProp(form):
# This is quicker than going through the norm process
nv = s_common.guid((p, v))
adds.append((i, 'node:ndef', nv, now))
if adds:
tot = len(adds)
logger.debug('Adding {:,d} node:ndef rows for [{}]'.format(
tot, form))
with self.core.getCoreXact() as xact:
i = 0
nt = 100000
for chunk in s_common.chunks(adds, nt):
self.core.store.addRows(chunk)
i = i + len(chunk)
logger.debug(
'Loading {:,d} [{}%] rows into transaction'.format(
i, int((i / tot) * 100)))
logger.debug('Processed {:,d} [{}%] forms.'.format(
n, int((n / nforms) * 100)))
logger.debug('Finished adding node:ndef rows to the Cortex')
async def __anit__(self, boss, task, name, user, info=None, root=None):
await s_base.Base.__anit__(self)
if info is None:
info = {}
self.boss = boss
task._syn_task = self
self.task = task # the real task...
self.iden = s_common.guid()
self.tick = s_common.now()
self.boss.tasks[self.iden] = self
if root is not None:
root.kids[self.iden] = self
self.task.add_done_callback(self._onTaskDone)
self.name = name
self.user = user
self.root = root
self.info = info
self.kids = {}
self.onfini(self._onTaskFini)
def testmain():
pconf = {'user': '******', 'passwd': 'root'}
with self.getTestProxy(dmon, 'core', **pconf) as core:
# Setup user permissions
core.addAuthRole('creator')
core.addAuthRule('creator', (True, ('node:add', )))
core.addAuthRule('creator', (True, ('prop:set', )))
core.addAuthRule('creator', (True, ('tag:add', )))
core.addUserRole('root', 'creator')
host, port = dmon.addr
curl = f'tcp://*****:*****@{host}:{port}/core'
guid = s_common.guid()
seen = s_common.now()
gestdef = self.getIngestDef(guid, seen)
with self.getTestDir() as dirn:
# Test yaml support here
gestfp = s_common.genpath(dirn, 'gest.yaml')
s_common.yamlsave(gestdef, gestfp)
argv = [
'--cortex', curl, '--debug', '--modules',
'synapse.tests.utils.TestModule', gestfp
]
outp = self.getTestOutp()
cmdg = s_t_utils.CmdGenerator(['storm pivcomp -> *'],
on_end=EOFError)
with mock.patch('synapse.lib.cli.get_input', cmdg):
self.eq(s_feed.main(argv, outp=outp), 0)
self.true(outp.expect('teststr=haha', throw=False))
self.true(outp.expect('pivtarg=hehe', throw=False))
async def tryPasswd(self, passwd):
if self.info.get('locked', False):
return False
if passwd is None:
return False
onepass = self.info.get('onepass')
if onepass is not None:
expires, salt, hashed = onepass
if expires >= s_common.now():
if s_common.guid((salt, passwd)) == hashed:
await self.auth.setUserInfo(self.iden, 'onepass', None)
return True
else:
await self.auth.setUserInfo(self.iden, 'onepass', None)
shadow = self.info.get('passwd')
if shadow is None:
return False
salt, hashed = shadow
if s_common.guid((salt, passwd)) == hashed:
return True
return False
async def revModel20210312(self, layers):
ipv4type = self.core.model.type('inet:ipv4')
ipv6type = self.core.model.type('inet:ipv6')
for layr in layers:
nodeedits = []
meta = {
'time': s_common.now(),
'user': self.core.auth.rootuser.iden
}
async def save():
await layr.storNodeEdits(nodeedits, meta)
nodeedits.clear()
async for buid, propvalu in layr.iterPropRows(
'inet:web:acct', 'signup:client:ipv6'):
ipv6text = ipv6type.norm(ipv4type.repr(propvalu))[0]
nodeedits.append((buid, 'inet:web:acct',
((s_layer.EDIT_PROP_SET,
('signup:client:ipv6', ipv6text, propvalu,
s_layer.STOR_TYPE_IPV6), ()), )), )
if len(nodeedits) >= 1000:
await save()
if nodeedits:
await save()
def genCertTokn(rpub, **info):
'''
Generate a public key certificate token.
Args:
rpub (s_ecc.PubKey):
**info: Additional key/value data to be added to the certificate token.
Returns:
bytes: A msgpack encoded dictionary.
'''
tick = s_common.now()
info['ecdsa:pubkey'] = rpub.dump()
info['created'] = s_common.now()
info.setdefault('expires', tick + (3 * s_const.year))
return s_msgpack.en(info)
async def __anit__(self, boss, task, name, user, info=None, root=None):
await s_base.Base.__anit__(self)
if info is None:
info = {}
self.boss = boss
task._syn_task = self
self.task = task # the real task...
self.iden = s_common.guid()
self.tick = s_common.now()
self.boss.tasks[self.iden] = self
if root is not None:
root.kids[self.iden] = self
self.task.add_done_callback(self._onTaskDone)
self.name = name
self.user = user
self.root = root
self.info = info
self.kids = {}
self.onfini(self._onTaskFini)
async def _setAssignee(self, valu):
self.proj.confirm(('project', 'ticket', 'set', 'assignee'))
strvalu = await tostr(valu, noneok=True)
if strvalu is None:
await self.node.pop('assignee')
await self.node.set('updated', s_common.now())
return
udef = await self.proj.runt.snap.core.getUserDefByName(strvalu)
if udef is None:
mesg = f'No user found by the name {strvalu}'
raise s_exc.NoSuchUser(mesg=mesg)
await self.node.set('assignee', udef['iden'])
await self.node.set('updated', s_common.now())
async def _setSprint(self, valu):
self.proj.confirm(('project', 'ticket', 'set', 'sprint'))
strvalu = await tostr(valu, noneok=True)
if strvalu is None:
await self.node.pop('sprint')
await self.node.set('updated', s_common.now())
return
sprint = await self.proj._getProjSprint(strvalu)
if sprint is None:
mesg = f'No sprint found by that name/iden ({strvalu}).'
raise s_exc.NoSuchName(mesg=mesg)
await self.node.set('sprint', sprint.node.ndef[1])
await self.node.set('updated', s_common.now())
def convert_ipynb(_):
import synapse.common as s_common
import nbconvert.nbconvertapp as nba
cwd = os.getcwd()
for fdir, dirs, fns in os.walk(cwd):
if '.ipynb_checkpoints' in dirs:
dirs.remove('.ipynb_checkpoints')
for fn in fns:
if fn.endswith('.ipynb'):
# if 'httpapi' not in fn:
# continue
tick = s_common.now()
fp = os.path.join(fdir, fn)
args = ['--execute', '--template', './vertex.tpl', '--to', 'rst', fp]
nba.main(args)
tock = s_common.now()
took = (tock - tick) / 1000
print(f'convert_ipynb: Notebook {fn} execution took {took} seconds.')
async def runAxonTestBase(self, axon):
tick = s_common.now()
self.false(await axon.has(asdfhash))
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
self.eq(asdfretn, await fd.save())
# do it again to test the short circuit
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
self.eq(asdfretn, await fd.save())
bytz = []
async for byts in axon.get(asdfhash):
bytz.append(byts)
self.eq(b'asdfasdf', b''.join(bytz))
self.true(await axon.has(asdfhash))
self.false(await axon.has(bbufhash))
self.eq((bbufhash, ), await axon.wants((bbufhash, asdfhash)))
async with await axon.upload() as fd:
await fd.write(bbuf)
await fd.save()
self.true(await axon.has(asdfhash))
self.true(await axon.has(bbufhash))
self.eq((), await axon.wants((bbufhash, asdfhash)))
items = [x async for x in axon.hashes(0)]
self.eq(((0, (asdfhash, 8)), (1, (bbufhash, 33554437))), items)
items = [x[1] async for x in axon.history(tick)]
self.eq(((asdfhash, 8), (bbufhash, 33554437)), items)
items = [x[1] async for x in axon.history(0, tock=1)]
self.eq((), items)
info = await axon.metrics()
self.eq(33554445, info.get('size:bytes'))
self.eq(2, info.get('file:count'))
async with await axon.upload() as fd:
await fd.write(b'')
self.eq(emptyretn, await fd.save())
bytz = []
async for byts in axon.get(emptyhash):
bytz.append(byts)
self.eq(b'', b''.join(bytz))
def _normRelStr(self, valu, relto=None):
valu = valu.strip().lower()
# assumes the relative string starts with a - or +
delt = s_time.delta(valu)
if not relto:
relto = s_common.now()
return self.timetype._normPyInt(delt + relto)[0]
async def runAxonTestBase(self, axon):
tick = s_common.now()
self.false(await axon.has(asdfhash))
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
self.eq(asdfretn, await fd.save())
# do it again to test the short circuit
async with await axon.upload() as fd:
await fd.write(b'asdfasdf')
self.eq(asdfretn, await fd.save())
bytz = []
async for byts in axon.get(asdfhash):
bytz.append(byts)
self.eq(b'asdfasdf', b''.join(bytz))
self.true(await axon.has(asdfhash))
self.false(await axon.has(bbufhash))
self.eq((bbufhash,), await axon.wants((bbufhash, asdfhash)))
async with await axon.upload() as fd:
await fd.write(bbuf)
await fd.save()
self.true(await axon.has(asdfhash))
self.true(await axon.has(bbufhash))
self.eq((), await axon.wants((bbufhash, asdfhash)))
items = [x async for x in axon.hashes(0)]
self.eq(((0, (asdfhash, 8)), (1, (bbufhash, 33554437))), items)
items = [x[1] async for x in axon.history(tick)]
self.eq(((asdfhash, 8), (bbufhash, 33554437)), items)
items = [x[1] async for x in axon.history(0, tock=1)]
self.eq((), items)
info = await axon.metrics()
self.eq(33554445, info.get('size:bytes'))
self.eq(2, info.get('file:count'))
async with await axon.upload() as fd:
await fd.write(b'')
self.eq(emptyretn, await fd.save())
bytz = []
async for byts in axon.get(emptyhash):
bytz.append(byts)
self.eq(b'', b''.join(bytz))
async def _setStatus(self, valu):
useriden = self.proj.runt.user.iden
if useriden != self.node.get('assignee'):
self.proj.confirm(('project', 'ticket', 'set', 'status'))
strvalu = await tostr(valu)
await self.node.set('status', strvalu)
await self.node.set('updated', s_common.now())
def _setRowsByIdProp(self, iden, prop, valu):
if s_compat.isint(valu):
count = self.update(self._q_uprows_by_iden_prop_int, iden=iden, prop=prop, valu=valu)
else:
count = self.update(self._q_uprows_by_iden_prop_str, iden=iden, prop=prop, valu=valu)
if count == 0:
rows = [(iden, prop, valu, s_common.now()), ]
self._addRows(rows)
def puts(self, items):
'''
Add the structured data from items to the CryoTank.
Args:
items (list): A list of objects to store in the CryoTank.
Returns:
int: The index that the item storage began at.
'''
itembyts = [s_msgpack.en(i) for i in items]
tick = s_common.now()
bytesize = sum([len(b) for b in itembyts])
with self.lmdb.begin(db=self.lmdb_items, write=True) as xact:
retn = self.items_indx
todo = []
for byts in itembyts:
todo.append((struct.pack('>Q', self.items_indx), byts))
self.items_indx += 1
with xact.cursor() as curs:
curs.putmulti(todo, append=True)
took = s_common.now() - tick
with xact.cursor(db=self.lmdb_metrics) as curs:
lkey = struct.pack('>Q', self.metrics_indx)
self.metrics_indx += 1
info = {
'time': tick,
'count': len(items),
'size': bytesize,
'took': took
}
curs.put(lkey, s_msgpack.en(info), append=True)
return retn
def forcecommit(self):
'''
Note:
This method may raise a MapFullError
'''
if not self.dirty:
return False
xactopslen = len(self.xactops)
# ok... lets commit and re-open
starttime = s_common.now()
self._finiCoXact()
donetime = s_common.now()
self.commitstats.append((starttime, xactopslen, donetime - starttime))
self._initCoXact()
return True
def backup(srcdir, dstdir):
tick = s_common.now()
srcdir = s_common.reqdir(srcdir)
dstdir = s_common.gendir(dstdir)
logger.info(f'Starting backup of [{srcdir}]')
logger.info(f'Destination dir: [{dstdir}]')
for root, dnames, fnames in os.walk(srcdir, topdown=True):
relpath = os.path.relpath(root, start=srcdir)
for name in list(dnames):
srcpath = s_common.genpath(root, name)
dstpath = s_common.genpath(dstdir, relpath, name)
if name.endswith('.lmdb'):
dnames.remove(name)
backup_lmdb(srcpath, dstpath)
continue
logger.info(f'making dir:{dstpath}')
s_common.gendir(dstpath)
for name in fnames:
srcpath = s_common.genpath(root, name)
# skip unix sockets etc...
if not os.path.isfile(srcpath):
continue
dstpath = s_common.genpath(dstdir, relpath, name)
logger.info(f'copying: {srcpath} -> {dstpath}')
shutil.copy(srcpath, dstpath)
tock = s_common.now()
logger.info(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
return
async def _initTestRunts(self):
modl = self.core.model
fnme = 'test:runt'
form = modl.form(fnme)
now = s_common.now()
data = [(' BEEP ', {'tick': modl.type('time').norm('2001')[0], 'lulz': 'beep.sys', '.created': now}),
('boop', {'tick': modl.type('time').norm('2010')[0], '.created': now}),
('blah', {'tick': modl.type('time').norm('2010')[0], 'lulz': 'blah.sys'}),
('woah', {}),
]
for pprop, propd in data:
props = {}
pnorm, _ = form.type.norm(pprop)
for k, v in propd.items():
prop = form.props.get(k)
if prop:
norm, _ = prop.type.norm(v)
props[k] = norm
props.setdefault('.created', s_common.now())
rows = [('*' + fnme, pnorm)]
for k, v in props.items():
rows.append((k, v))
buid = s_common.buid((fnme, pnorm))
self._runtsByBuid[buid] = rows
# Allow for indirect lookup to a set of buids
self._runtsByPropValu[fnme].append(buid)
self._runtsByPropValu[(fnme, pnorm)].append(buid)
for k, propvalu in props.items():
prop = fnme + ':' + k
if k.startswith('.'):
prop = fnme + k
self._runtsByPropValu[prop].append(buid)
if modl.prop(prop).type.indx(propvalu):
# Can the secondary property be indexed for lift?
self._runtsByPropValu[(prop, propvalu)].append(buid)
async def stor(self, sops, splices=None):
if not splices:
await self.wlyr.stor(sops)
return
now = s_common.now()
user = self.user.iden
wasnew, providen, provstack = self.core.provstor.commit()
if wasnew:
await self.fire('prov:new', time=now, user=user, prov=providen, provstack=provstack)
for splice in splices:
name, info = splice
info.update(time=now, user=user, prov=providen)
await self.fire(name, **info)
await self.wlyr.stor(sops, splices=splices)
async def test_ival(self):
model = s_datamodel.Model()
ival = model.types.get('ival')
self.eq(b'', ival.indx(None))
self.eq(('2016/01/01 00:00:00.000', '2017/01/01 00:00:00.000'), ival.repr(ival.norm(('2016', '2017'))[0]))
self.gt(s_common.now(), ival._normRelStr('-1 min'))
self.eq((0, 5356800000), ival.norm((0, '1970-03-04'))[0])
self.eq((1451606400000, 1451606400001), ival.norm('2016')[0])
self.eq((1451606400000, 1451606400001), ival.norm(1451606400000)[0])
self.eq((1451606400000, 1451606400001), ival.norm('2016')[0])
self.eq((1451606400000, 1483228800000), ival.norm(('2016', ' 2017'))[0])
self.eq((1451606400000, 1483228800000), ival.norm(('2016-01-01', ' 2017'))[0])
self.eq((1451606400000, 1483142400000), ival.norm(('2016', '+365 days'))[0])
self.eq((1448150400000, 1451606400000), ival.norm(('2016', '-40 days'))[0])
self.eq((1447891200000, 1451347200000), ival.norm(('2016-3days', '-40 days '))[0])
self.eq((1451347200000, 0x7fffffffffffffff), ival.norm(('2016-3days', '?'))[0])
start = s_common.now() + s_time.oneday - 1
end = ival.norm(('now', '+1day'))[0][1]
self.lt(start, end)
oldv = ival.norm(('2016', '2017'))[0]
newv = ival.norm(('2015', '2018'))[0]
self.eq((1420070400000, 1514764800000), ival.merge(oldv, newv))
self.raises(s_exc.BadTypeValu, ival.norm, '?')
self.raises(s_exc.BadTypeValu, ival.norm, ('', ''))
self.raises(s_exc.BadTypeValu, ival.norm, ('2016-3days', '+77days', '-40days'))
self.raises(s_exc.BadTypeValu, ival.norm, ('?', '-1 day'))
async with self.getTestCore() as core:
t = core.model.type('test:time')
async with await core.snap() as snap:
node = await snap.addNode('test:str', 'a', {'tick': '2014', '.seen': ('2005', '2006')})
await node.addTag('foo', valu=('2000', '2001'))
node = await snap.addNode('test:str', 'b', {'tick': '2015', '.seen': ('8679', '9000')})
await node.addTag('foo', valu=('2015', '2018'))
node = await snap.addNode('test:str', 'c', {'tick': '2016', '.seen': ('now-5days', 'now-1day')})
await node.addTag('bar', valu=('1970', '1990'))
node = await snap.addNode('test:str', 'd', {'tick': 'now', '.seen': ('now-10days', '?')})
await node.addTag('baz', valu='now')
node = await snap.addNode('test:str', 'e', {'tick': 'now-3days', '.seen': ('now+1day', 'now+5days')})
await node.addTag('biz', valu=('now-1day', 'now+1day'))
# node whose primary prop is an ival
node = await snap.addNode('test:ival', (0, 10), {'interval': ("now", "now+4days")})
node = await snap.addNode('test:ival', (50, 100), {'interval': ("now-2days", "now+2days")})
node = await snap.addNode('test:ival', ("1995", "1997"), {'interval': ("2010", "2011")})
node = await snap.addNode('test:ival', ("now-2days", "now+4days"), {'interval': ("201006", "20100605")})
node = await snap.addNode('test:ival', ("now+21days", "?"), {'interval': ("2000", "2001")})
# tag of tags
node = (await alist(snap.getNodesBy('syn:tag', 'foo')))[0]
await node.addTag('v.p', valu=('2005', '2006'))
node = (await alist(snap.getNodesBy('syn:tag', 'bar')))[0]
await node.addTag('vert.proj', valu=('20110605', 'now'))
node = (await alist(snap.getNodesBy('syn:tag', 'biz')))[0]
await node.addTag('vertex.project', valu=('now-5days', 'now'))
await self.agenraises(s_exc.BadSyntax, core.eval('test:str :tick=(20150102, "-4 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=(now, "-1 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=2015'))
await self.agenlen(1, core.eval('test:str +:tick@=(2015, "+1 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=(20150102+1day, "-4 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=(20150102, "-4 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=(now, "-1 day")'))
await self.agenlen(1, core.eval('test:str +:tick@=("now-1day", "?")'))
await self.agenlen(1, core.eval('test:str +:tick@=("now+2days", "-3 day")'))
await self.agenlen(0, core.eval('test:str +:tick@=("now", "now+3days")'))
await self.agenlen(1, core.eval('test:str +:tick@=("now-2days","now")'))
await self.agenlen(0, core.eval('test:str +:tick@=("2011", "2014")'))
await self.agenlen(1, core.eval('test:str +:tick@=("2014", "20140601")'))
await self.agenlen(1, core.eval('test:str:tick@=(now, "-1 day")'))
await self.agenlen(1, core.eval('test:str:tick@=2015'))
await self.agenlen(1, core.eval('test:str:tick@=(2015, "+1 day")'))
await self.agenlen(1, core.eval('test:str:tick@=(20150102+1day, "-4 day")'))
await self.agenlen(1, core.eval('test:str:tick@=(20150102, "-4 day")'))
await self.agenlen(1, core.eval('test:str:tick@=(now, "-1 day")'))
await self.agenlen(1, core.eval('test:str:tick@=("now-1day", "?")'))
await self.agenlen(1, core.eval('test:str:tick@=("now+2days", "-3 day")'))
await self.agenlen(0, core.eval('test:str:tick@=("now", "now+3days")'))
await self.agenlen(1, core.eval('test:str:tick@=("now-2days","now")'))
await self.agenlen(0, core.eval('test:str:tick@=("2011", "2014")'))
await self.agenlen(1, core.eval('test:str:tick@=("2014", "20140601")'))
await self.agenlen(0, core.eval('.seen@=("2004", "2005")'))
await self.agenlen(1, core.eval('.seen@=("9000", "9001")'))
await self.agenlen(2, core.eval('.seen@=("now+6days", "?")'))
await self.agenlen(2, core.eval('.seen@=(now, "-4 days")'))
await self.agenlen(2, core.eval('.seen@=(8900, 9500)'))
await self.agenlen(1, core.eval('.seen@=("2004", "20050201")'))
await self.agenlen(2, core.eval('.seen@=("now", "-3 days")'))
await self.agenlen(1, core.eval('test:ival@=1970'))
await self.agenlen(5, core.eval('test:ival@=(1970, "now+100days")'))
await self.agenlen(1, core.eval('test:ival@="now"'))
await self.agenlen(1, core.eval('test:ival@=("now+1day", "now+6days")'))
await self.agenlen(1, core.eval('test:ival@=("now-9days", "now-1day")'))
await self.agenlen(1, core.eval('test:ival@=("now-3days", "now+3days")'))
await self.agenlen(0, core.eval('test:ival@=("1993", "1995")'))
await self.agenlen(0, core.eval('test:ival@=("1997", "1998")'))
await self.agenlen(1, core.eval('test:ival:interval@="now+2days"'))
await self.agenlen(0, core.eval('test:ival:interval@=("now-4days","now-3days")'))
await self.agenlen(0, core.eval('test:ival:interval@=("now+4days","now+6days")'))
await self.agenlen(1, core.eval('test:ival:interval@=("now-3days","now-1days")'))
await self.agenlen(1, core.eval('test:ival:interval@=("now+3days","now+6days")'))
await self.agenlen(2, core.eval('test:ival:interval@="now+1day"'))
await self.agenlen(2, core.eval('test:ival:interval@=("20100602","20100603")'))
await self.agenlen(2, core.eval('test:ival:interval@=("now-10days","now+10days")'))
await self.agenlen(0, core.eval('test:ival:interval@=("1999", "2000")'))
await self.agenlen(0, core.eval('test:ival:interval@=("2001", "2002")'))
await self.agenlen(1, core.eval('test:ival +:interval@="now+2days"'))
await self.agenlen(0, core.eval('test:ival +:interval@=("now-4days","now-3days")'))
await self.agenlen(0, core.eval('test:ival +:interval@=("now+4days","now+6days")'))
await self.agenlen(1, core.eval('test:ival +:interval@=("now-3days","now-1days")'))
await self.agenlen(1, core.eval('test:ival +:interval@=("now+3days","now+6days")'))
await self.agenlen(2, core.eval('test:ival +:interval@="now+1day"'))
await self.agenlen(2, core.eval('test:ival +:interval@=("20100602","20100603")'))
await self.agenlen(2, core.eval('test:ival +:interval@=("now-10days","now+10days")'))
await self.agenlen(0, core.eval('test:ival +:interval@=("1999", "2000")'))
await self.agenlen(0, core.eval('test:ival +:interval@=("2001", "2002")'))
await self.agenlen(0, core.eval('#foo@=("2013", "2015")'))
await self.agenlen(0, core.eval('#foo@=("2018", "2019")'))
await self.agenlen(1, core.eval('#foo@=("1999", "2002")'))
await self.agenlen(1, core.eval('#foo@="2015"'))
await self.agenlen(1, core.eval('#foo@=("2010", "20150601")'))
await self.agenlen(2, core.eval('#foo@=("2000", "2017")'))
await self.agenlen(1, core.eval('#bar@=("1985", "1995")'))
await self.agenlen(0, core.eval('#bar@="2000"'))
await self.agenlen(1, core.eval('#baz@=("now","-1 day")'))
await self.agenlen(1, core.eval('#baz@=("now-1day", "+1day")'))
await self.agenlen(1, core.eval('#biz@="now"'))
await self.agenlen(0, core.eval('#foo +#foo@=("2013", "2015")'))
await self.agenlen(0, core.eval('#foo +#foo@=("2018", "2019")'))
await self.agenlen(1, core.eval('#foo +#foo@=("1999", "2002")'))
await self.agenlen(1, core.eval('#foo +#foo@="2015"'))
await self.agenlen(1, core.eval('#foo +#foo@=("2010", "20150601")'))
await self.agenlen(2, core.eval('#foo +#foo@=("2000", "2017")'))
await self.agenlen(1, core.eval('#bar +#bar@=("1985", "1995")'))
await self.agenlen(0, core.eval('#bar +#bar@="2000"'))
await self.agenlen(1, core.eval('#baz +#baz@=("now","-1 day")'))
await self.agenlen(1, core.eval('#baz +#baz@=("now-1day", "+1day")'))
await self.agenlen(1, core.eval('#biz +#biz@="now"'))
await self.agenlen(0, core.eval('test:str#foo@=("2013", "2015")'))
await self.agenlen(0, core.eval('test:str#foo@=("2018", "2019")'))
await self.agenlen(1, core.eval('test:str#foo@=("1999", "2002")'))
await self.agenlen(1, core.eval('test:str#foo@="2015"'))
await self.agenlen(1, core.eval('test:str#foo@=("2010", "20150601")'))
await self.agenlen(2, core.eval('test:str#foo@=("2000", "2017")'))
await self.agenlen(1, core.eval('test:str#bar@=("1985", "1995")'))
await self.agenlen(0, core.eval('test:str#bar@="2000"'))
await self.agenlen(1, core.eval('test:str#baz@=("now","-1 day")'))
await self.agenlen(1, core.eval('test:str#baz@=("now-1day", "+1day")'))
await self.agenlen(1, core.eval('test:str#biz@="now"'))
await self.agenlen(0, core.eval('test:str +#foo@=("2013", "2015")'))
await self.agenlen(0, core.eval('test:str +#foo@=("2018", "2019")'))
await self.agenlen(1, core.eval('test:str +#foo@=("1999", "2002")'))
await self.agenlen(1, core.eval('test:str +#foo@="2015"'))
await self.agenlen(1, core.eval('test:str +#foo@=("2010", "20150601")'))
await self.agenlen(2, core.eval('test:str +#foo@=("2000", "2017")'))
await self.agenlen(1, core.eval('test:str +#bar@=("1985", "1995")'))
await self.agenlen(0, core.eval('test:str +#bar@="2000"'))
await self.agenlen(1, core.eval('test:str +#baz@=("now","-1 day")'))
await self.agenlen(1, core.eval('test:str +#baz@=("now-1day", "+1day")'))
await self.agenlen(1, core.eval('test:str +#biz@="now"'))
await self.agenlen(0, core.eval('##v.p@=("2003", "2005")'))
await self.agenlen(0, core.eval('##v.p@=("2006", "2008")'))
await self.agenlen(1, core.eval('##vert.proj@="2016"'))
await self.agenlen(1, core.eval('##vert.proj@=("2010", "2012")'))
await self.agenlen(1, core.eval('##vert.proj@=("2016", "now+6days")'))
await self.agenlen(1, core.eval('##vert.proj@=("1995", "now+6 days")'))
await self.agenlen(1, core.eval('##vertex.project@=("now-9days", "now-3days")'))
await self.agenlen(0, core.eval('test:str +:tick@=(2020, 2000)'))
now = s_common.now()
nodes = await alist(core.eval('[test:guid="*" .seen=("-1 day","?")]'))
node = nodes[0]
valu = node.get('.seen')
self.eq(valu[1], ival.futsize)
self.true(now - s_const.day <= valu[0] < now)
# Sad Paths
q = '[test:str=newp .seen=(2018/03/31,2018/03/30)]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
q = '[test:str=newp .seen=("+-1 day","+-1 day")]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
q = '[test:str=newp .seen=("?","?")]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
q = '[test:str=newp .seen=(2018/03/31,2018/03/31)]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
q = '[test:str=newp .seen=(2008, 2019, 2000)]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
q = '[test:str=newp .seen=("?","-1 day")]'
await self.agenraises(s_exc.BadPropValu, core.eval(q))
# *range= not supported for ival
q = 'test:str +:.seen*range=((20090601, 20090701), (20110905, 20110906,))'
await self.agenraises(s_exc.NoSuchCmpr, core.eval(q))
q = 'test:str.seen*range=((20090601, 20090701), (20110905, 20110906,))'
await self.agenraises(s_exc.NoSuchCmpr, core.eval(q))
async def _addNodeFnibOps(self, fnib, editatom, props=None):
'''
Add a node via (form, norm, info, buid) and add ops to editatom
'''
form, norm, info, buid = fnib
if form.isrunt:
raise s_exc.IsRuntForm(mesg='Cannot make runt nodes.',
form=form.full, prop=norm)
if props is None:
props = {}
# Check if this buid is already under construction
node = editatom.getNodeBeingMade(buid)
if node is not None:
return node
# Check if this buid is already fully made
node = await self.getNodeByBuid(buid)
if node is not None:
return node
# Another editatom might have created in another task during the above call, so check again
node = editatom.getNodeBeingMade(buid)
if node is not None:
return node
if props is None:
props = {}
# lets build a node...
node = s_node.Node(self, None)
node.buid = buid
node.form = form
node.ndef = (form.name, norm)
sops = form.getSetOps(buid, norm)
editatom.sops.extend(sops)
editatom.addNode(node)
# update props with any subs from form value
subs = info.get('subs')
if subs is not None:
for name, valu in subs.items():
if form.prop(name) is not None:
props[name] = valu
# update props with any defvals we are missing
for name, valu in form.defvals.items():
props.setdefault(name, valu)
# set all the properties with init=True
for name, valu in props.items():
await node._setops(name, valu, editatom, init=True)
# set our global properties
tick = s_common.now()
await node._setops('.created', tick, editatom, init=True)
return None
async def test_it_forms_hostexec(self):
# forms related to the host execution model
async with self.getTestCore() as core:
async with await core.snap() as snap:
exe = 'sha256:' + 'a' * 64
port = 80
tick = s_common.now()
host = s_common.guid()
proc = s_common.guid()
mutex = 'giggleXX_X0'
pipe = 'pipe\\mynamedpipe'
user = '******'
pid = 20
key = 'HKEY_LOCAL_MACHINE\\Foo\\Bar'
ipv4 = 0x01020304
ipv6 = '::1'
addr4 = f'tcp://1.2.3.4:{port}'
addr6 = f'udp://[::1]:{port}'
url = 'http://www.google.com/sekrit.html'
raw_path = r'c:\Windows\System32\rar.exe'
norm_path = r'c:/windows/system32/rar.exe'
src_proc = s_common.guid()
src_path = r'c:/temp/ping.exe'
cmd0 = 'rar a -r yourfiles.rar *.txt'
fpath = 'c:/temp/yourfiles.rar'
fbyts = 'sha256:' + 'b' * 64
pprops = {
'exe': exe,
'pid': pid,
'cmd': cmd0,
'host': host,
'time': tick,
'user': user,
'path': raw_path,
'src:exe': src_path,
'src:proc': src_proc,
}
node = await snap.addNode('it:exec:proc', proc, pprops)
self.eq(node.ndef[1], proc)
self.eq(node.get('exe'), exe)
self.eq(node.get('pid'), pid)
self.eq(node.get('cmd'), cmd0)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('user'), user)
self.eq(node.get('path'), norm_path)
self.eq(node.get('src:exe'), src_path)
self.eq(node.get('src:proc'), src_proc)
m0 = s_common.guid()
mprops = {
'exe': exe,
'proc': proc,
'name': mutex,
'host': host,
'time': tick,
}
node = await snap.addNode('it:exec:mutex', m0, mprops)
self.eq(node.ndef[1], m0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('name'), mutex)
p0 = s_common.guid()
pipeprops = {
'exe': exe,
'proc': proc,
'name': pipe,
'host': host,
'time': tick,
}
node = await snap.addNode('it:exec:pipe', p0, pipeprops)
self.eq(node.ndef[1], p0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('name'), pipe)
u0 = s_common.guid()
uprops = {
'proc': proc,
'host': host,
'exe': exe,
'time': tick,
'url': url,
'client': addr4,
}
node = await snap.addNode('it:exec:url', u0, uprops)
self.eq(node.ndef[1], u0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('url'), url)
self.eq(node.get('client'), addr4)
self.eq(node.get('client:ipv4'), ipv4)
self.eq(node.get('client:port'), port)
u1 = s_common.guid()
uprops['client'] = addr6
node = await snap.addNode('it:exec:url', u1, uprops)
self.eq(node.ndef[1], u1)
self.eq(node.get('client'), addr6)
self.eq(node.get('client:ipv6'), ipv6)
self.eq(node.get('client:port'), port)
b0 = s_common.guid()
bprops = {
'proc': proc,
'host': host,
'exe': exe,
'time': tick,
'server': addr4
}
node = await snap.addNode('it:exec:bind', b0, bprops)
self.eq(node.ndef[1], b0)
self.eq(node.get('exe'), exe)
self.eq(node.get('proc'), proc)
self.eq(node.get('host'), host)
self.eq(node.get('time'), tick)
self.eq(node.get('server'), addr4)
self.eq(node.get('server:ipv4'), ipv4)
self.eq(node.get('server:port'), port)
b1 = s_common.guid()
bprops['server'] = addr6
node = await snap.addNode('it:exec:bind', b1, bprops)
self.eq(node.ndef[1], b1)
self.eq(node.get('server'), addr6)
self.eq(node.get('server:ipv6'), ipv6)
self.eq(node.get('server:port'), port)
faprops = {
'exe': exe,
'host': host,
'proc': proc,
'file': fbyts,
'time': tick,
'path': fpath,
}
fa0 = s_common.guid()
node = await snap.addNode('it:exec:file:add', fa0, faprops)
self.eq(node.ndef[1], fa0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fr0 = s_common.guid()
node = await snap.addNode('it:exec:file:read', fr0, faprops)
self.eq(node.ndef[1], fr0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fw0 = s_common.guid()
node = await snap.addNode('it:exec:file:write', fw0, faprops)
self.eq(node.ndef[1], fw0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
fd0 = s_common.guid()
node = await snap.addNode('it:exec:file:del', fd0, faprops)
self.eq(node.ndef[1], fd0)
self.eq(node.get('exe'), exe)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('time'), tick)
self.eq(node.get('file'), fbyts)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
file0 = s_common.guid()
fsprops = {
'host': host,
'path': fpath,
'file': fbyts,
'ctime': tick,
'mtime': tick + 1,
'atime': tick + 2,
'user': user,
'group': 'domainadmin'
}
node = await snap.addNode('it:fs:file', file0, fsprops)
self.eq(node.ndef[1], file0)
self.eq(node.get('host'), host)
self.eq(node.get('user'), user)
self.eq(node.get('group'), 'domainadmin')
self.eq(node.get('file'), fbyts)
self.eq(node.get('ctime'), tick)
self.eq(node.get('mtime'), tick + 1)
self.eq(node.get('atime'), tick + 2)
self.eq(node.get('path'), fpath)
self.eq(node.get('path:dir'), 'c:/temp')
self.eq(node.get('path:base'), 'yourfiles.rar')
self.eq(node.get('path:ext'), 'rar')
# FIXME - This test would be cleaner with stable guid generation
rprops = {
'host': host,
'proc': proc,
'exe': exe,
'time': tick,
'reg': '*',
}
forms = ('it:exec:reg:get',
'it:exec:reg:set',
'it:exec:reg:del',
)
for form in forms:
rk0 = s_common.guid()
nprops = rprops.copy()
node = await snap.addNode(form, rk0, nprops)
self.eq(node.ndef[1], rk0)
self.eq(node.get('host'), host)
self.eq(node.get('proc'), proc)
self.eq(node.get('exe'), exe)
self.eq(node.get('time'), tick)
self.nn(node.get('reg'))
async def test_time(self):
model = s_datamodel.Model()
ttime = model.types.get('time')
self.gt(s_common.now(), ttime.norm('-1hour')[0])
tminmax = ttime.clone({'min': True, 'max': True})
# Merge testing with tminmax
now = s_common.now()
self.eq(now + 1, tminmax.merge(now, now + 1))
self.eq(now, tminmax.merge(now + 1, now))
async with self.getTestCore() as core:
t = core.model.type('test:time')
# explicitly test our "future/ongoing" value...
future = 0x7fffffffffffffff
self.eq(t.indx(future), b'\xff\xff\xff\xff\xff\xff\xff\xff')
self.eq(t.norm('?')[0], future)
self.eq(t.norm(future)[0], future)
self.eq(t.repr(future), '?')
# Explicitly test our max time vs. future marker
maxtime = 253402300799999 # 9999/12/31 23:59:59.999
self.eq(t.norm(maxtime)[0], maxtime)
self.eq(t.repr(maxtime), '9999/12/31 23:59:59.999')
self.eq(t.norm('9999/12/31 23:59:59.999')[0], maxtime)
self.raises(s_exc.BadTypeValu, t.norm, maxtime + 1)
tick = t.norm('2014')[0]
self.eq(t.repr(tick), '2014/01/01 00:00:00.000')
tock = t.norm('2015')[0]
self.raises(s_exc.BadCmprValu,
t.cmpr, '2015', '*range=', tick)
async with await core.snap() as snap:
node = await snap.addNode('test:str', 'a', {'tick': '2014'})
node = await snap.addNode('test:str', 'b', {'tick': '2015'})
node = await snap.addNode('test:str', 'c', {'tick': '2016'})
node = await snap.addNode('test:str', 'd', {'tick': 'now'})
nodes = await alist(core.getNodesBy('test:str:tick', '2014'))
self.eq({node.ndef[1] for node in nodes}, {'a'})
nodes = await alist(core.getNodesBy('test:str:tick', ('2014', '2015'),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'a', 'b'})
nodes = await alist(core.getNodesBy('test:str:tick', '201401*'))
self.eq({node.ndef[1] for node in nodes}, {'a'})
nodes = await alist(core.getNodesBy('test:str:tick', ('-3000 days', 'now'),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'a', 'b', 'c', 'd'})
nodes = await alist(core.getNodesBy('test:str:tick', (tick, tock),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'a', 'b'})
nodes = await alist(core.getNodesBy('test:str:tick', ('20131231', '+2 days'),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'a'})
nodes = await alist(core.eval('test:str:tick*range=(20131231, "+2 days")'))
self.eq({node.ndef[1] for node in nodes}, {'a'})
nodes = await alist(core.getNodesBy('test:str:tick', ('-1 day', '+1 day'),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'d'})
nodes = await alist(core.getNodesBy('test:str:tick', ('-1 days', 'now', ),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'d'})
# Equivalent lift
nodes = await alist(core.getNodesBy('test:str:tick', ('now', '-1 days'),
cmpr='*range='))
self.eq({node.ndef[1] for node in nodes}, {'d'})
# This is equivalent of the previous lift
self.eq({node.ndef[1] for node in nodes}, {'d'})
# Sad path
self.raises(s_exc.NoSuchFunc, t.indxByEq, ('', ''))
self.raises(s_exc.NoSuchFunc, t.indxByEq, ('?', '-1 day'))
self.true(t.cmpr('2015', '>=', '20140202'))
self.true(t.cmpr('2015', '>=', '2015'))
self.true(t.cmpr('2015', '>', '20140202'))
self.false(t.cmpr('2015', '>', '2015'))
self.true(t.cmpr('20150202', '<=', '2016'))
self.true(t.cmpr('20150202', '<=', '2016'))
self.true(t.cmpr('20150202', '<', '2016'))
self.false(t.cmpr('2015', '<', '2015'))
await self.agenlen(1, core.eval('test:str +:tick=2015'))
await self.agenlen(1, core.eval('test:str +:tick*range=($test, "+- 2day")',
opts={'vars': {'test': '2015'}}))
await self.agenlen(1, core.eval('test:str +:tick*range=(now, "-+ 1day")'))
await self.agenlen(1, core.eval('test:str +:tick*range=(2015, "+1 day")'))
await self.agenlen(1, core.eval('test:str +:tick*range=(20150102, "-3 day")'))
await self.agenlen(0, core.eval('test:str +:tick*range=(20150201, "+1 day")'))
await self.agenlen(1, core.eval('test:str +:tick*range=(20150102, "+- 2day")'))
await self.agenlen(2, core.eval('test:str +:tick*range=(2015, 2016)'))
await self.agenlen(0, core.eval('test:str +:tick*range=(2016, 2015)'))
await self.agenlen(2, core.eval('test:str:tick*range=(2015, 2016)'))
await self.agenlen(0, core.eval('test:str:tick*range=(2016, 2015)'))
await self.agenlen(1, core.eval('test:str:tick*range=(2015, "+1 day")'))
await self.agenlen(4, core.eval('test:str:tick*range=(2014, "now")'))
await self.agenlen(0, core.eval('test:str:tick*range=(20150201, "+1 day")'))
await self.agenlen(1, core.eval('test:str:tick*range=(now, "+-1 day")'))
await self.agenlen(0, core.eval('test:str:tick*range=(now, "+1 day")'))
# Sad path for *range=
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str:tick*range=("+- 1day", "now")'))
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str:tick*range=("-+ 1day", "now")'))
await self.agenraises(s_exc.BadPropValu,
core.eval('[test:guid="*" :tick="+-1 day"]'))
await self.agenraises(s_exc.BadCmprValu,
core.eval('test:str:tick*range=(2015)'))
await self.agenraises(s_exc.BadCmprValu,
core.getNodesBy('test:str:tick', tick, '*range='))
await self.agenraises(s_exc.BadCmprValu,
core.eval('test:str +:tick*range=(2015)'))
await self.agenraises(s_exc.BadCmprValu,
core.eval('test:str +:tick*range=(2015, 2016, 2017)'))
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str +:tick*range=("?", "+1 day")'))
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str +:tick*range=(2000, "?+1 day")'))
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str:tick*range=("?", "+1 day")'))
await self.agenraises(s_exc.BadTypeValu,
core.eval('test:str:tick*range=(2000, "?+1 day")'))
async with await core.snap() as snap:
node = await snap.addNode('test:str', 't1', {'tick': '2018/12/02 23:59:59.000'})
node = await snap.addNode('test:str', 't2', {'tick': '2018/12/03'})
node = await snap.addNode('test:str', 't3', {'tick': '2018/12/03 00:00:01.000'})
await self.agenlen(0, core.eval('test:str:tick*range=(2018/12/01, "+24 hours")'))
await self.agenlen(2, core.eval('test:str:tick*range=(2018/12/01, "+48 hours")'))
await self.agenlen(0, core.eval('test:str:tick*range=(2018/12/02, "+23 hours")'))
await self.agenlen(1, core.eval('test:str:tick*range=(2018/12/02, "+86399 seconds")'))
await self.agenlen(2, core.eval('test:str:tick*range=(2018/12/02, "+24 hours")'))
await self.agenlen(3, core.eval('test:str:tick*range=(2018/12/02, "+86401 seconds")'))
await self.agenlen(3, core.eval('test:str:tick*range=(2018/12/02, "+25 hours")'))
async with self.getTestCore() as core:
async with await core.snap() as snap:
node = await snap.addNode('test:str', 'a', {'tick': '2014'})
node = await snap.addNode('test:int', node.get('tick') + 1)
node = await snap.addNode('test:str', 'b', {'tick': '2015'})
node = await snap.addNode('test:int', node.get('tick') + 1)
node = await snap.addNode('test:str', 'c', {'tick': '2016'})
node = await snap.addNode('test:int', node.get('tick') + 1)
node = await snap.addNode('test:str', 'd', {'tick': 'now'})
node = await snap.addNode('test:int', node.get('tick') + 1)
q = 'test:int $end=$node.value() test:str:tick*range=(2015, $end) -test:int'
nodes = await alist(core.eval(q))
self.len(6, nodes)
self.eq({node.ndef[1] for node in nodes}, {'b', 'c', 'd'})
def add(self, item):
tick = s_common.now()
lkey = tick.to_bytes(8, 'big')
self.slab.put(lkey, s_msgpack.en(item), dupdata=True, db=self.db)
