def hexstr(text):
'''
Ensure a string is valid hex.
Args:
text (str): String to normalize.
Examples:
Norm a few strings:
hexstr('0xff00')
hexstr('ff00')
Notes:
Will accept strings prefixed by '0x' or '0X' and remove them.
Returns:
str: Normalized hex string.
'''
text = text.strip().lower()
if text.startswith(('0x', '0X')):
text = text[2:]
if not text:
raise s_exc.BadTypeValu(valu=text, name='hexstr',
mesg='No string left after stripping')
try:
# checks for valid hex width and does character
# checking in C without using regex
s_common.uhex(text)
except (binascii.Error, ValueError) as e:
raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg=str(e))
return text
def hexstr(text):
'''
Ensure a string is valid hex.
Args:
text (str): String to normalize.
Examples:
Norm a few strings:
hexstr('0xff00')
hexstr('ff00')
Notes:
Will accept strings prefixed by '0x' or '0X' and remove them.
Returns:
str: Normalized hex string.
'''
text = text.strip().lower()
if text.startswith(('0x', '0X')):
text = text[2:]
if not text:
raise s_exc.BadTypeValu(valu=text,
name='hexstr',
mesg='No string left after stripping')
try:
# checks for valid hex width and does character
# checking in C without using regex
s_common.uhex(text)
except (binascii.Error, ValueError) as e:
raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg=str(e))
return text
def _normPyStr(self, valu):
if valu == '*':
guid = s_common.guid()
norm = f'guid:{guid}'
return norm, {}
if valu.find(':') == -1:
# we're ok with un-adorned sha256s
if len(valu) == 64 and s_common.uhex(valu):
valu = valu.lower()
subs = {'sha256': valu}
return f'sha256:{valu}', {'subs': subs}
raise s_exc.BadTypeValu(
name=self.name,
valu=valu,
mesg='unadorned file:bytes value is not a sha256')
kind, kval = valu.split(':', 1)
if kind == 'base64':
byts = base64.b64decode(kval)
return self._normPyBytes(byts)
kval = kval.lower()
if kind == 'hex':
byts = s_common.uhex(kval)
return self._normPyBytes(byts)
if kind == 'guid':
kval = kval.lower()
if not s_common.isguid(kval):
raise s_exc.BadTypeValu(name=self.name,
valu=valu,
mesg='guid is not a guid')
return f'guid:{kval}', {}
if kind == 'sha256':
if len(kval) != 64:
raise s_exc.BadTypeValu(name=self.name,
valu=valu,
mesg='invalid length for sha256 valu')
s_common.uhex(kval)
subs = {'sha256': kval}
return f'sha256:{kval}', {'subs': subs}
raise s_exc.BadTypeValu(name=self.name,
valu=valu,
kind=kind,
mesg='unable to norm as file:bytes')
async def test_link_rx_sadpath(self):
junk = 'a9d0cdafef705b9864bd'
# random sequence of data which causes an error
# to be thrown when unpacking data via msgpack.
junk = s_common.uhex(junk)
evt = asyncio.Event()
async def onlink(link):
await link.tx(('what', {'k': 1}))
self.true(await s_coro.event_wait(evt, 6))
# Send purposely bad data through the link
await link.send(junk)
await link.fini()
serv = await s_link.listen('127.0.0.1', 0, onlink)
host, port = serv.sockets[0].getsockname()
link = await s_link.connect(host, port)
msg0 = await link.rx()
self.eq(msg0, ('what', {'k': 1}))
evt.set()
await asyncio.sleep(0)
with self.getAsyncLoggerStream('synapse.lib.link',
'rx error') as stream:
msg1 = await link.rx()
self.true(await stream.wait(6))
self.none(msg1)
async def _load_all(self):
'''
Load all the appointments from persistent storage
'''
to_delete = []
for idenf, val in self._hivedict.items():
try:
iden = s_common.uhex(idenf)
appt = _Appt.unpack(val)
if appt.iden != iden:
raise s_exc.InconsistentStorage(mesg='iden inconsistency')
self._addappt(iden, appt)
self._next_indx = max(self._next_indx, appt.indx + 1)
except (s_exc.InconsistentStorage, s_exc.BadTime, TypeError,
KeyError) as e:
logger.warning(
'Invalid appointment %r found in storage: %r. Removing',
iden, e)
to_delete.append(iden)
continue
for iden in to_delete:
await self._hivedict.pop(s_common.ehex(iden))
# Make sure we don't assign the same index to 2 appointments
if self.appts:
maxindx = max(appt.indx for appt in self.appts.values())
self._next_indx = maxindx + 1
async def test_storm_lib_bytes(self):
async with self.getTestCore() as core:
with self.raises(s_exc.BadArg):
opts = {'vars': {'bytes': 10}}
text = '($size, $sha2) = $lib.bytes.put($bytes)'
nodes = await core.nodes(text, opts=opts)
opts = {'vars': {'bytes': b'asdfasdf'}}
text = '($size, $sha2) = $lib.bytes.put($bytes) [ test:int=$size test:str=$sha2 ]'
nodes = await core.nodes(text, opts=opts)
self.len(2, nodes)
self.eq(nodes[0].ndef, ('test:int', 8))
self.eq(nodes[1].ndef, (
'test:str',
'2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892'
))
bkey = s_common.uhex(
'2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892'
)
byts = b''.join([b async for b in core.axon.get(bkey)])
self.eq(b'asdfasdf', byts)
async def main(argv, outp=None):
pars = setup()
opts = pars.parse_args(argv)
path = s_common.getSynPath('telepath.yaml')
telefini = await s_telepath.loadTeleEnv(path)
if outp is None: # pragma: no cover
outp = s_output.OutPut()
if opts.output is None:
opts.output = '.'
outdir = pathlib.Path(opts.output)
s_common.gendir(opts.output)
async with await s_telepath.openurl(opts.axon) as axon:
# reminder: these are the hashes *not* available
awants = await axon.wants([s_common.uhex(h) for h in opts.hashes])
for a in awants:
outp.printf(f'{s_common.ehex(a)} not in axon store')
exists = [h for h in opts.hashes if s_common.uhex(h) not in awants]
for h in exists:
try:
outp.printf(f'Fetching {h} to file')
with open(outdir.joinpath(h), 'wb') as fd:
async for b in axon.get(s_common.uhex(h)):
fd.write(b)
outp.printf(f'Fetched {h} to file')
except Exception as e:
outp.printf('Error: Hit Exception: %s' % (str(e), ))
continue
if telefini: # pragma: no cover
await telefini()
return 0
async def iterMpkFile(self, sha256):
'''
Yield items from a .mpk message pack stream file.
'''
unpk = s_msgpack.Unpk()
async for byts in self.get(s_common.uhex(sha256)):
for _, item in unpk.feed(byts):
yield item
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.slab.get(buid, db=self.db)
if byts is None:
return 0
return int.from_bytes(byts, byteorder='big')
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.slab.get(buid, db=self.db)
if byts is None:
return 0
return int.from_bytes(byts, byteorder='big')
def test_common_ehex_uhex(self):
byts = b'deadb33f00010203'
s = s_common.ehex(byts)
self.isinstance(s, str)
self.eq(s, '64656164623333663030303130323033')
# uhex is a linear transform back
obyts = s_common.uhex(s)
self.isinstance(obyts, bytes)
self.eq(byts, obyts)
async def getLyrErrs(self, lyriden):
lpref = s_common.uhex(lyriden)
errs = []
for lkey, errb in self.slab.scanByPref(lpref, db=self.errors):
offset = s_common.int64un(lkey[16:])
err = s_msgpack.un(errb)
errs.append((offset, err))
return errs
def test_common_ehex_uhex(self):
byts = b'deadb33f00010203'
s = s_common.ehex(byts)
self.isinstance(s, str)
self.eq(s, '64656164623333663030303130323033')
# uhex is a linear transform back
obyts = s_common.uhex(s)
self.isinstance(obyts, bytes)
self.eq(byts, obyts)
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.lenv.get(buid, db=self.db)
if byts is None:
return 0
return s_common.int64un(byts)
async def _setLyrErr(self, lyriden, offset, err):
self.errcnts.inc(lyriden)
lkey = s_common.uhex(lyriden) + s_common.int64en(offset)
errb = s_msgpack.en(err)
return self.slab.put(lkey,
errb,
dupdata=False,
overwrite=True,
db=self.errors)
def get(self, iden):
buid = s_common.uhex(iden)
byts = self.lenv.get(buid, db=self.db)
if byts is None:
return 0
return s_common.int64un(byts)
def test_str(self):
model = s_datamodel.Model()
lowr = model.type('str').clone({'lower': True})
self.eq('foo', lowr.norm('FOO')[0])
self.eq((('pref', b'bhaha'), ), lowr.indxByPref('BHAHA'))
self.eq(True, lowr.cmpr('xxherexx', '~=', 'here'))
self.eq(False, lowr.cmpr('xxherexx', '~=', '^here'))
self.eq(True, lowr.cmpr('foo', '!=', 'bar'))
self.eq(False, lowr.cmpr('foo', '!=', 'FOO'))
self.eq(True, lowr.cmpr('foobar', '^=', 'FOO'))
self.eq(False, lowr.cmpr('foubar', '^=', 'FOO'))
regx = model.type('str').clone({'regex': '^[a-f][0-9]+$'})
self.eq('a333', regx.norm('a333')[0])
self.raises(s_exc.BadTypeValu, regx.norm, 'A333')
regl = model.type('str').clone({
'regex': '^[a-f][0-9]+$',
'lower': True
})
self.eq('a333', regl.norm('a333')[0])
self.eq('a333', regl.norm('A333')[0])
self.eq(b'haha', model.type('str').indx('haha'))
byts = s_common.uhex('e2889e')
self.eq(byts, model.type('str').indx('∞'))
# The real world is a harsh place.
self.eq(b'haha\xed\xb3\xbe hehe',
model.type('str').indx('haha\udcfe hehe'))
self.eq(b'haha\xed\xb3\xbe ',
model.type('str').indxByPref('haha\udcfe ')[0][1])
strp = model.type('str').clone({'strip': True})
self.eq('foo', strp.norm(' foo \t')[0])
self.eq(b'foo bar', strp.indxByPref(' foo bar')[0][1])
self.eq(b'foo bar ', strp.indxByPref(' foo bar ')[0][1])
onespace = model.type('str').clone({'onespace': True})
self.eq('foo', onespace.norm(' foo\t')[0])
self.eq('hehe haha', onespace.norm('hehe haha')[0])
self.eq(b'foo', onespace.indxByPref(' foo')[0][1])
self.eq(b'foo bar', onespace.indxByPref(' foo bar')[0][1])
self.eq(b'foo bar', onespace.indxByPref(' foo bar ')[0][1])
self.eq(b'foo ba', onespace.indxByPref(' foo ba')[0][1])
enums = model.type('str').clone({'enums': 'hehe,haha,zork'})
self.eq('hehe', enums.norm('hehe')[0])
self.eq('haha', enums.norm('haha')[0])
self.eq('zork', enums.norm('zork')[0])
self.raises(s_exc.BadTypeValu, enums.norm, 'zing')
def test_str(self):
model = s_datamodel.Model()
lowr = model.type('str').clone({'lower': True})
self.eq('foo', lowr.norm('FOO')[0])
self.eq((('pref', b'bhaha'),), lowr.indxByPref('BHAHA'))
self.eq(True, lowr.cmpr('xxherexx', '~=', 'here'))
self.eq(False, lowr.cmpr('xxherexx', '~=', '^here'))
self.eq(True, lowr.cmpr('foo', '!=', 'bar'))
self.eq(False, lowr.cmpr('foo', '!=', 'FOO'))
self.eq(True, lowr.cmpr('foobar', '^=', 'FOO'))
self.eq(False, lowr.cmpr('foubar', '^=', 'FOO'))
regx = model.type('str').clone({'regex': '^[a-f][0-9]+$'})
self.eq('a333', regx.norm('a333')[0])
self.raises(s_exc.BadTypeValu, regx.norm, 'A333')
regl = model.type('str').clone({'regex': '^[a-f][0-9]+$', 'lower': True})
self.eq('a333', regl.norm('a333')[0])
self.eq('a333', regl.norm('A333')[0])
self.eq(b'haha', model.type('str').indx('haha'))
byts = s_common.uhex('e2889e')
self.eq(byts, model.type('str').indx('∞'))
# The real world is a harsh place.
self.eq(b'haha\xed\xb3\xbe hehe', model.type('str').indx('haha\udcfe hehe'))
self.eq(b'haha\xed\xb3\xbe ', model.type('str').indxByPref('haha\udcfe ')[0][1])
strp = model.type('str').clone({'strip': True})
self.eq('foo', strp.norm(' foo \t')[0])
self.eq(b'foo bar', strp.indxByPref(' foo bar')[0][1])
self.eq(b'foo bar ', strp.indxByPref(' foo bar ')[0][1])
onespace = model.type('str').clone({'onespace': True})
self.eq('foo', onespace.norm(' foo\t')[0])
self.eq('hehe haha', onespace.norm('hehe haha')[0])
self.eq(b'foo', onespace.indxByPref(' foo')[0][1])
self.eq(b'foo bar', onespace.indxByPref(' foo bar')[0][1])
self.eq(b'foo bar', onespace.indxByPref(' foo bar ')[0][1])
self.eq(b'foo ba', onespace.indxByPref(' foo ba')[0][1])
enums = model.type('str').clone({'enums': 'hehe,haha,zork'})
self.eq('hehe', enums.norm('hehe')[0])
self.eq('haha', enums.norm('haha')[0])
self.eq('zork', enums.norm('zork')[0])
self.raises(s_exc.BadTypeValu, enums.norm, 'zing')
strsubs = model.type('str').clone({'regex': r'(?P<first>[ab]+)(?P<last>[zx]+)'})
norm, info = strsubs.norm('aabbzxxxxxz')
self.eq(info.get('subs'), {'first': 'aabb', 'last': 'zxxxxxz'})
async def iterUserNotifs(self, useriden, size=None):
# iterate user notifications backward
userbyts = s_common.uhex(useriden)
count = 0
for _, indxbyts in self.slab.scanByPrefBack(userbyts, db=self.notif_indx_usertype):
indx = s_common.int64un(indxbyts)
mesg = self.notifseqn.getraw(indxbyts)
yield (indx, mesg)
count += 1
if size is not None and count >= size:
break
async def delete(self, sha256):
if not await self.allowed(('axon', 'del')):
return
sha256b = s_common.uhex(sha256)
if not await self.cell.has(sha256b):
self.set_status(404)
self.sendRestErr('NoSuchFile', f'SHA-256 not found: {sha256}')
return
resp = await self.cell.del_(sha256b)
return self.sendRestRetn(resp)
async def post(self):
if not await self.allowed(('axon', 'del')):
return
body = self.getJsonBody(validator=reqValidAxonDel)
if body is None:
return
sha256s = body.get('sha256s')
hashes = [s_common.uhex(s) for s in sha256s]
resp = await self.cell.dels(hashes)
return self.sendRestRetn(tuple(zip(sha256s, resp)))
async def iterMpkFile(self, sha256):
'''
Yield items from a MsgPack (.mpk) file in the Axon.
Args:
sha256 (str): The sha256 hash of the file as a string.
Yields:
Unpacked items from the bytes.
'''
unpk = s_msgpack.Unpk()
async for byts in self.get(s_common.uhex(sha256)):
for _, item in unpk.feed(byts):
yield item
async def _save(self):
size, sha256b = await self.upfd.save()
fhashes = {
htyp: hasher.hexdigest()
for htyp, hasher in self.hashset.hashes
}
assert sha256b == s_common.uhex(fhashes.get('sha256'))
assert size == self.hashset.size
fhashes['size'] = size
return self.sendRestRetn(fhashes)
async def delUserNotif(self, indx):
envl = self.notifseqn.pop(indx)
if envl is None:
return
mesg = envl[1]
useriden, mesgtime, mesgtype, mesgdata = mesg
indxbyts = s_common.int64en(indx)
userbyts = s_common.uhex(useriden)
timebyts = s_common.int64en(mesgtime)
typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode())
self.slab.delete(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime)
self.slab.delete(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype)
async def _addUserNotif(self, mesg, nexsitem):
indx = self.notifseqn.add(mesg, indx=nexsitem[0])
indxbyts = s_common.int64en(indx)
useriden, mesgtime, mesgtype, mesgdata = mesg
userbyts = s_common.uhex(useriden)
timebyts = s_common.int64en(mesgtime)
typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode())
self.slab.put(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime, dupdata=True)
self.slab.put(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype, dupdata=True)
return indx
async def setNodeProp(self, iden, name, valu):
buid = s_common.uhex(iden)
async with await self.cell.snap(user=self.user) as snap:
node = await snap.getNodeByBuid(buid)
if node is None:
raise s_exc.NoSuchIden(iden=iden)
prop = node.form.props.get(name)
self._reqUserAllowed('prop:set', prop.full)
await node.set(name, valu)
return node.pack()
async def readlines(self, sha256):
remain = ''
async for byts in self.get(s_common.uhex(sha256)):
text = remain + byts.decode()
lines = text.split('\n')
if len(lines) == 1:
remain = text
continue
remain = lines[-1]
for line in lines[:-1]:
yield line
if remain:
yield remain
async def execStormCmd(self, runt, genr):
async for x in genr:
yield x
for iden in self.opts.iden:
try:
buid = s_common.uhex(iden)
except Exception:
await runt.warn(f'Failed to decode iden: [{iden}]')
continue
if len(buid) != 32:
await runt.warn(f'iden must be 32 bytes [{iden}]')
continue
node = await runt.snap.getNodeByBuid(buid)
if node is not None:
yield node, runt.initPath(node)
async def getInput(self):
for node in self.inputs:
yield node, self.initPath(node)
for ndef in self.opts.get('ndefs', ()):
node = await self.snap.getNodeByNdef(ndef)
if node is not None:
yield node, self.initPath(node)
for iden in self.opts.get('idens', ()):
buid = s_common.uhex(iden)
if len(buid) != 32:
raise s_exc.NoSuchIden(mesg='Iden must be 32 bytes', iden=iden)
node = await self.snap.getNodeByBuid(buid)
if node is not None:
yield node, self.initPath(node)
async def delNodeTag(self, iden, tag):
'''
Delete a tag from the node specified by iden.
Args:
iden (str): A hex encoded node BUID.
tag (str): A tag string.
'''
buid = s_common.uhex(iden)
parts = tag.split('.')
self._reqUserAllowed('tag:del', *parts)
async with await self.cell.snap(user=self.user) as snap:
node = await snap.getNodeByBuid(buid)
if node is None:
raise s_exc.NoSuchIden(iden=iden)
await node.delTag(tag)
return node.pack()
async def get(self, sha256):
if not await self.reqAuthAllowed(('axon', 'get')):
return
sha256b = s_common.uhex(sha256)
self.set_header('Content-Type', 'application/octet-stream')
self.set_header('Content-Disposition', 'attachment')
try:
async for byts in self.cell.get(sha256b):
self.write(byts)
await self.flush()
await asyncio.sleep(0)
except s_exc.NoSuchFile as e:
self.set_status(404)
self.sendRestErr('NoSuchFile', e.get('mesg'))
return
async def addNodeTag(self, iden, tag, valu=(None, None)):
'''
Add a tag to a node specified by iden.
Args:
iden (str): A hex encoded node BUID.
tag (str): A tag string.
valu (tuple): A time interval tuple or (None, None).
'''
buid = s_common.uhex(iden)
parts = tag.split('.')
self._reqUserAllowed('tag:add', *parts)
async with await self.cell.snap(user=self.user) as snap:
node = await snap.getNodeByBuid(buid)
if node is None:
raise s_exc.NoSuchIden(iden=iden)
await node.addTag(tag, valu=valu)
return node.pack()
def delete(self, iden):
buid = s_common.uhex(iden)
self.lenv.delete(buid, db=self.db)
def set(self, iden, offs):
buid = s_common.uhex(iden)
byts = s_common.int64en(offs)
self.lenv.put(buid, byts, db=self.db)
async def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
path = s_common.getSynPath('telepath.yaml')
telefini = await s_telepath.loadTeleEnv(path)
pars = makeargparser()
opts = pars.parse_args(argv)
axon = await s_telepath.openurl(opts.axon)
core = None
if opts.cortex:
core = await s_telepath.openurl(opts.cortex)
tags = set()
if opts.tags:
for tag in opts.tags.split(','):
tags.add(tag)
tags = tuple(tags)
if tags:
outp.printf(f'adding tags: {tags}')
filepaths = set()
for item in opts.filenames:
paths = glob.glob(item, recursive=opts.recursive)
if not paths:
outp.printf(f'filepath does not contain any files: {item}')
continue
filepaths.update([path for path in paths if os.path.isfile(path)])
for path in filepaths:
bname = os.path.basename(path)
hset = s_hashset.HashSet()
with s_common.reqfile(path) as fd:
hset.eatfd(fd)
fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
sha256 = fhashes.get('sha256')
bsha256 = s_common.uhex(sha256)
if not await axon.has(bsha256):
async with await axon.upload() as upfd:
with s_common.genfile(path) as fd:
for byts in s_common.iterfd(fd):
await upfd.write(byts)
size, hashval = await upfd.save()
if hashval != bsha256: # pragma: no cover
raise s_exc.SynErr(mesg='hashes do not match',
ehash=s_common.ehex(hashval),
ahash=hashval)
outp.printf(f'Uploaded [{bname}] to axon')
else:
outp.printf(f'Axon already had [{bname}]')
if core:
opts = {
'vars': {
'md5': fhashes.get('md5'),
'sha1': fhashes.get('sha1'),
'sha256': fhashes.get('sha256'),
'size': hset.size,
'name': bname,
'tags': tags,
}
}
q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \
'{ for $tag in $tags { [+#$tag] } }'
msgs = await core.storm(q, opts=opts).list()
node = [m[1] for m in msgs if m[0] == 'node'][0]
iden = node[0][1]
size = node[1]['props']['size']
name = node[1]['props']['name']
mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
outp.printf(mesg)
await axon.fini()
if core:
await core.fini()
if telefini: # pragma: no cover
await telefini()
return 0
def set(self, iden, offs):
buid = s_common.uhex(iden)
byts = s_common.int64en(offs)
self.lenv.put(buid, byts, db=self.db)
def gen(self, iden):
bidn = s_common.uhex(iden)
return SlabDict(self.slab, db=self.db, pref=bidn)
def set(self, iden, offs):
buid = s_common.uhex(iden)
byts = offs.to_bytes(length=8, byteorder='big')
self.slab.put(buid, byts, db=self.db)
