def hexstr(text): ''' Ensure a string is valid hex. Args: text (str): String to normalize. Examples: Norm a few strings: hexstr('0xff00') hexstr('ff00') Notes: Will accept strings prefixed by '0x' or '0X' and remove them. Returns: str: Normalized hex string. ''' text = text.strip().lower() if text.startswith(('0x', '0X')): text = text[2:] if not text: raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg='No string left after stripping') try: # checks for valid hex width and does character # checking in C without using regex s_common.uhex(text) except (binascii.Error, ValueError) as e: raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg=str(e)) return text
def hexstr(text): ''' Ensure a string is valid hex. Args: text (str): String to normalize. Examples: Norm a few strings: hexstr('0xff00') hexstr('ff00') Notes: Will accept strings prefixed by '0x' or '0X' and remove them. Returns: str: Normalized hex string. ''' text = text.strip().lower() if text.startswith(('0x', '0X')): text = text[2:] if not text: raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg='No string left after stripping') try: # checks for valid hex width and does character # checking in C without using regex s_common.uhex(text) except (binascii.Error, ValueError) as e: raise s_exc.BadTypeValu(valu=text, name='hexstr', mesg=str(e)) return text
def _normPyStr(self, valu): if valu == '*': guid = s_common.guid() norm = f'guid:{guid}' return norm, {} if valu.find(':') == -1: # we're ok with un-adorned sha256s if len(valu) == 64 and s_common.uhex(valu): valu = valu.lower() subs = {'sha256': valu} return f'sha256:{valu}', {'subs': subs} raise s_exc.BadTypeValu( name=self.name, valu=valu, mesg='unadorned file:bytes value is not a sha256') kind, kval = valu.split(':', 1) if kind == 'base64': byts = base64.b64decode(kval) return self._normPyBytes(byts) kval = kval.lower() if kind == 'hex': byts = s_common.uhex(kval) return self._normPyBytes(byts) if kind == 'guid': kval = kval.lower() if not s_common.isguid(kval): raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='guid is not a guid') return f'guid:{kval}', {} if kind == 'sha256': if len(kval) != 64: raise s_exc.BadTypeValu(name=self.name, valu=valu, mesg='invalid length for sha256 valu') s_common.uhex(kval) subs = {'sha256': kval} return f'sha256:{kval}', {'subs': subs} raise s_exc.BadTypeValu(name=self.name, valu=valu, kind=kind, mesg='unable to norm as file:bytes')
async def test_link_rx_sadpath(self): junk = 'a9d0cdafef705b9864bd' # random sequence of data which causes an error # to be thrown when unpacking data via msgpack. junk = s_common.uhex(junk) evt = asyncio.Event() async def onlink(link): await link.tx(('what', {'k': 1})) self.true(await s_coro.event_wait(evt, 6)) # Send purposely bad data through the link await link.send(junk) await link.fini() serv = await s_link.listen('127.0.0.1', 0, onlink) host, port = serv.sockets[0].getsockname() link = await s_link.connect(host, port) msg0 = await link.rx() self.eq(msg0, ('what', {'k': 1})) evt.set() await asyncio.sleep(0) with self.getAsyncLoggerStream('synapse.lib.link', 'rx error') as stream: msg1 = await link.rx() self.true(await stream.wait(6)) self.none(msg1)
async def _load_all(self): ''' Load all the appointments from persistent storage ''' to_delete = [] for idenf, val in self._hivedict.items(): try: iden = s_common.uhex(idenf) appt = _Appt.unpack(val) if appt.iden != iden: raise s_exc.InconsistentStorage(mesg='iden inconsistency') self._addappt(iden, appt) self._next_indx = max(self._next_indx, appt.indx + 1) except (s_exc.InconsistentStorage, s_exc.BadTime, TypeError, KeyError) as e: logger.warning( 'Invalid appointment %r found in storage: %r. Removing', iden, e) to_delete.append(iden) continue for iden in to_delete: await self._hivedict.pop(s_common.ehex(iden)) # Make sure we don't assign the same index to 2 appointments if self.appts: maxindx = max(appt.indx for appt in self.appts.values()) self._next_indx = maxindx + 1
async def test_storm_lib_bytes(self): async with self.getTestCore() as core: with self.raises(s_exc.BadArg): opts = {'vars': {'bytes': 10}} text = '($size, $sha2) = $lib.bytes.put($bytes)' nodes = await core.nodes(text, opts=opts) opts = {'vars': {'bytes': b'asdfasdf'}} text = '($size, $sha2) = $lib.bytes.put($bytes) [ test:int=$size test:str=$sha2 ]' nodes = await core.nodes(text, opts=opts) self.len(2, nodes) self.eq(nodes[0].ndef, ('test:int', 8)) self.eq(nodes[1].ndef, ( 'test:str', '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892' )) bkey = s_common.uhex( '2413fb3709b05939f04cf2e92f7d0897fc2596f9ad0b8a9ea855c7bfebaae892' ) byts = b''.join([b async for b in core.axon.get(bkey)]) self.eq(b'asdfasdf', byts)
async def main(argv, outp=None): pars = setup() opts = pars.parse_args(argv) path = s_common.getSynPath('telepath.yaml') telefini = await s_telepath.loadTeleEnv(path) if outp is None: # pragma: no cover outp = s_output.OutPut() if opts.output is None: opts.output = '.' outdir = pathlib.Path(opts.output) s_common.gendir(opts.output) async with await s_telepath.openurl(opts.axon) as axon: # reminder: these are the hashes *not* available awants = await axon.wants([s_common.uhex(h) for h in opts.hashes]) for a in awants: outp.printf(f'{s_common.ehex(a)} not in axon store') exists = [h for h in opts.hashes if s_common.uhex(h) not in awants] for h in exists: try: outp.printf(f'Fetching {h} to file') with open(outdir.joinpath(h), 'wb') as fd: async for b in axon.get(s_common.uhex(h)): fd.write(b) outp.printf(f'Fetched {h} to file') except Exception as e: outp.printf('Error: Hit Exception: %s' % (str(e), )) continue if telefini: # pragma: no cover await telefini() return 0
async def iterMpkFile(self, sha256): ''' Yield items from a .mpk message pack stream file. ''' unpk = s_msgpack.Unpk() async for byts in self.get(s_common.uhex(sha256)): for _, item in unpk.feed(byts): yield item
def get(self, iden): buid = s_common.uhex(iden) byts = self.slab.get(buid, db=self.db) if byts is None: return 0 return int.from_bytes(byts, byteorder='big')
def get(self, iden): buid = s_common.uhex(iden) byts = self.slab.get(buid, db=self.db) if byts is None: return 0 return int.from_bytes(byts, byteorder='big')
def test_common_ehex_uhex(self): byts = b'deadb33f00010203' s = s_common.ehex(byts) self.isinstance(s, str) self.eq(s, '64656164623333663030303130323033') # uhex is a linear transform back obyts = s_common.uhex(s) self.isinstance(obyts, bytes) self.eq(byts, obyts)
async def getLyrErrs(self, lyriden): lpref = s_common.uhex(lyriden) errs = [] for lkey, errb in self.slab.scanByPref(lpref, db=self.errors): offset = s_common.int64un(lkey[16:]) err = s_msgpack.un(errb) errs.append((offset, err)) return errs
def test_common_ehex_uhex(self): byts = b'deadb33f00010203' s = s_common.ehex(byts) self.isinstance(s, str) self.eq(s, '64656164623333663030303130323033') # uhex is a linear transform back obyts = s_common.uhex(s) self.isinstance(obyts, bytes) self.eq(byts, obyts)
def get(self, iden): buid = s_common.uhex(iden) byts = self.lenv.get(buid, db=self.db) if byts is None: return 0 return s_common.int64un(byts)
async def _setLyrErr(self, lyriden, offset, err): self.errcnts.inc(lyriden) lkey = s_common.uhex(lyriden) + s_common.int64en(offset) errb = s_msgpack.en(err) return self.slab.put(lkey, errb, dupdata=False, overwrite=True, db=self.errors)
def get(self, iden): buid = s_common.uhex(iden) byts = self.lenv.get(buid, db=self.db) if byts is None: return 0 return s_common.int64un(byts)
def test_str(self): model = s_datamodel.Model() lowr = model.type('str').clone({'lower': True}) self.eq('foo', lowr.norm('FOO')[0]) self.eq((('pref', b'bhaha'), ), lowr.indxByPref('BHAHA')) self.eq(True, lowr.cmpr('xxherexx', '~=', 'here')) self.eq(False, lowr.cmpr('xxherexx', '~=', '^here')) self.eq(True, lowr.cmpr('foo', '!=', 'bar')) self.eq(False, lowr.cmpr('foo', '!=', 'FOO')) self.eq(True, lowr.cmpr('foobar', '^=', 'FOO')) self.eq(False, lowr.cmpr('foubar', '^=', 'FOO')) regx = model.type('str').clone({'regex': '^[a-f][0-9]+$'}) self.eq('a333', regx.norm('a333')[0]) self.raises(s_exc.BadTypeValu, regx.norm, 'A333') regl = model.type('str').clone({ 'regex': '^[a-f][0-9]+$', 'lower': True }) self.eq('a333', regl.norm('a333')[0]) self.eq('a333', regl.norm('A333')[0]) self.eq(b'haha', model.type('str').indx('haha')) byts = s_common.uhex('e2889e') self.eq(byts, model.type('str').indx('∞')) # The real world is a harsh place. self.eq(b'haha\xed\xb3\xbe hehe', model.type('str').indx('haha\udcfe hehe')) self.eq(b'haha\xed\xb3\xbe ', model.type('str').indxByPref('haha\udcfe ')[0][1]) strp = model.type('str').clone({'strip': True}) self.eq('foo', strp.norm(' foo \t')[0]) self.eq(b'foo bar', strp.indxByPref(' foo bar')[0][1]) self.eq(b'foo bar ', strp.indxByPref(' foo bar ')[0][1]) onespace = model.type('str').clone({'onespace': True}) self.eq('foo', onespace.norm(' foo\t')[0]) self.eq('hehe haha', onespace.norm('hehe haha')[0]) self.eq(b'foo', onespace.indxByPref(' foo')[0][1]) self.eq(b'foo bar', onespace.indxByPref(' foo bar')[0][1]) self.eq(b'foo bar', onespace.indxByPref(' foo bar ')[0][1]) self.eq(b'foo ba', onespace.indxByPref(' foo ba')[0][1]) enums = model.type('str').clone({'enums': 'hehe,haha,zork'}) self.eq('hehe', enums.norm('hehe')[0]) self.eq('haha', enums.norm('haha')[0]) self.eq('zork', enums.norm('zork')[0]) self.raises(s_exc.BadTypeValu, enums.norm, 'zing')
def test_str(self): model = s_datamodel.Model() lowr = model.type('str').clone({'lower': True}) self.eq('foo', lowr.norm('FOO')[0]) self.eq((('pref', b'bhaha'),), lowr.indxByPref('BHAHA')) self.eq(True, lowr.cmpr('xxherexx', '~=', 'here')) self.eq(False, lowr.cmpr('xxherexx', '~=', '^here')) self.eq(True, lowr.cmpr('foo', '!=', 'bar')) self.eq(False, lowr.cmpr('foo', '!=', 'FOO')) self.eq(True, lowr.cmpr('foobar', '^=', 'FOO')) self.eq(False, lowr.cmpr('foubar', '^=', 'FOO')) regx = model.type('str').clone({'regex': '^[a-f][0-9]+$'}) self.eq('a333', regx.norm('a333')[0]) self.raises(s_exc.BadTypeValu, regx.norm, 'A333') regl = model.type('str').clone({'regex': '^[a-f][0-9]+$', 'lower': True}) self.eq('a333', regl.norm('a333')[0]) self.eq('a333', regl.norm('A333')[0]) self.eq(b'haha', model.type('str').indx('haha')) byts = s_common.uhex('e2889e') self.eq(byts, model.type('str').indx('∞')) # The real world is a harsh place. self.eq(b'haha\xed\xb3\xbe hehe', model.type('str').indx('haha\udcfe hehe')) self.eq(b'haha\xed\xb3\xbe ', model.type('str').indxByPref('haha\udcfe ')[0][1]) strp = model.type('str').clone({'strip': True}) self.eq('foo', strp.norm(' foo \t')[0]) self.eq(b'foo bar', strp.indxByPref(' foo bar')[0][1]) self.eq(b'foo bar ', strp.indxByPref(' foo bar ')[0][1]) onespace = model.type('str').clone({'onespace': True}) self.eq('foo', onespace.norm(' foo\t')[0]) self.eq('hehe haha', onespace.norm('hehe haha')[0]) self.eq(b'foo', onespace.indxByPref(' foo')[0][1]) self.eq(b'foo bar', onespace.indxByPref(' foo bar')[0][1]) self.eq(b'foo bar', onespace.indxByPref(' foo bar ')[0][1]) self.eq(b'foo ba', onespace.indxByPref(' foo ba')[0][1]) enums = model.type('str').clone({'enums': 'hehe,haha,zork'}) self.eq('hehe', enums.norm('hehe')[0]) self.eq('haha', enums.norm('haha')[0]) self.eq('zork', enums.norm('zork')[0]) self.raises(s_exc.BadTypeValu, enums.norm, 'zing') strsubs = model.type('str').clone({'regex': r'(?P<first>[ab]+)(?P<last>[zx]+)'}) norm, info = strsubs.norm('aabbzxxxxxz') self.eq(info.get('subs'), {'first': 'aabb', 'last': 'zxxxxxz'})
async def iterUserNotifs(self, useriden, size=None): # iterate user notifications backward userbyts = s_common.uhex(useriden) count = 0 for _, indxbyts in self.slab.scanByPrefBack(userbyts, db=self.notif_indx_usertype): indx = s_common.int64un(indxbyts) mesg = self.notifseqn.getraw(indxbyts) yield (indx, mesg) count += 1 if size is not None and count >= size: break
async def delete(self, sha256): if not await self.allowed(('axon', 'del')): return sha256b = s_common.uhex(sha256) if not await self.cell.has(sha256b): self.set_status(404) self.sendRestErr('NoSuchFile', f'SHA-256 not found: {sha256}') return resp = await self.cell.del_(sha256b) return self.sendRestRetn(resp)
async def post(self): if not await self.allowed(('axon', 'del')): return body = self.getJsonBody(validator=reqValidAxonDel) if body is None: return sha256s = body.get('sha256s') hashes = [s_common.uhex(s) for s in sha256s] resp = await self.cell.dels(hashes) return self.sendRestRetn(tuple(zip(sha256s, resp)))
async def iterMpkFile(self, sha256): ''' Yield items from a MsgPack (.mpk) file in the Axon. Args: sha256 (str): The sha256 hash of the file as a string. Yields: Unpacked items from the bytes. ''' unpk = s_msgpack.Unpk() async for byts in self.get(s_common.uhex(sha256)): for _, item in unpk.feed(byts): yield item
async def _save(self): size, sha256b = await self.upfd.save() fhashes = { htyp: hasher.hexdigest() for htyp, hasher in self.hashset.hashes } assert sha256b == s_common.uhex(fhashes.get('sha256')) assert size == self.hashset.size fhashes['size'] = size return self.sendRestRetn(fhashes)
async def delUserNotif(self, indx): envl = self.notifseqn.pop(indx) if envl is None: return mesg = envl[1] useriden, mesgtime, mesgtype, mesgdata = mesg indxbyts = s_common.int64en(indx) userbyts = s_common.uhex(useriden) timebyts = s_common.int64en(mesgtime) typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode()) self.slab.delete(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime) self.slab.delete(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype)
async def _addUserNotif(self, mesg, nexsitem): indx = self.notifseqn.add(mesg, indx=nexsitem[0]) indxbyts = s_common.int64en(indx) useriden, mesgtime, mesgtype, mesgdata = mesg userbyts = s_common.uhex(useriden) timebyts = s_common.int64en(mesgtime) typeabrv = self.notif_abrv_type.setBytsToAbrv(mesgtype.encode()) self.slab.put(userbyts + timebyts, indxbyts, db=self.notif_indx_usertime, dupdata=True) self.slab.put(userbyts + typeabrv + timebyts, indxbyts, db=self.notif_indx_usertype, dupdata=True) return indx
async def setNodeProp(self, iden, name, valu): buid = s_common.uhex(iden) async with await self.cell.snap(user=self.user) as snap: node = await snap.getNodeByBuid(buid) if node is None: raise s_exc.NoSuchIden(iden=iden) prop = node.form.props.get(name) self._reqUserAllowed('prop:set', prop.full) await node.set(name, valu) return node.pack()
async def readlines(self, sha256): remain = '' async for byts in self.get(s_common.uhex(sha256)): text = remain + byts.decode() lines = text.split('\n') if len(lines) == 1: remain = text continue remain = lines[-1] for line in lines[:-1]: yield line if remain: yield remain
async def execStormCmd(self, runt, genr): async for x in genr: yield x for iden in self.opts.iden: try: buid = s_common.uhex(iden) except Exception: await runt.warn(f'Failed to decode iden: [{iden}]') continue if len(buid) != 32: await runt.warn(f'iden must be 32 bytes [{iden}]') continue node = await runt.snap.getNodeByBuid(buid) if node is not None: yield node, runt.initPath(node)
async def getInput(self): for node in self.inputs: yield node, self.initPath(node) for ndef in self.opts.get('ndefs', ()): node = await self.snap.getNodeByNdef(ndef) if node is not None: yield node, self.initPath(node) for iden in self.opts.get('idens', ()): buid = s_common.uhex(iden) if len(buid) != 32: raise s_exc.NoSuchIden(mesg='Iden must be 32 bytes', iden=iden) node = await self.snap.getNodeByBuid(buid) if node is not None: yield node, self.initPath(node)
async def delNodeTag(self, iden, tag): ''' Delete a tag from the node specified by iden. Args: iden (str): A hex encoded node BUID. tag (str): A tag string. ''' buid = s_common.uhex(iden) parts = tag.split('.') self._reqUserAllowed('tag:del', *parts) async with await self.cell.snap(user=self.user) as snap: node = await snap.getNodeByBuid(buid) if node is None: raise s_exc.NoSuchIden(iden=iden) await node.delTag(tag) return node.pack()
async def get(self, sha256): if not await self.reqAuthAllowed(('axon', 'get')): return sha256b = s_common.uhex(sha256) self.set_header('Content-Type', 'application/octet-stream') self.set_header('Content-Disposition', 'attachment') try: async for byts in self.cell.get(sha256b): self.write(byts) await self.flush() await asyncio.sleep(0) except s_exc.NoSuchFile as e: self.set_status(404) self.sendRestErr('NoSuchFile', e.get('mesg')) return
async def addNodeTag(self, iden, tag, valu=(None, None)): ''' Add a tag to a node specified by iden. Args: iden (str): A hex encoded node BUID. tag (str): A tag string. valu (tuple): A time interval tuple or (None, None). ''' buid = s_common.uhex(iden) parts = tag.split('.') self._reqUserAllowed('tag:add', *parts) async with await self.cell.snap(user=self.user) as snap: node = await snap.getNodeByBuid(buid) if node is None: raise s_exc.NoSuchIden(iden=iden) await node.addTag(tag, valu=valu) return node.pack()
def delete(self, iden): buid = s_common.uhex(iden) self.lenv.delete(buid, db=self.db)
def set(self, iden, offs): buid = s_common.uhex(iden) byts = s_common.int64en(offs) self.lenv.put(buid, byts, db=self.db)
async def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() path = s_common.getSynPath('telepath.yaml') telefini = await s_telepath.loadTeleEnv(path) pars = makeargparser() opts = pars.parse_args(argv) axon = await s_telepath.openurl(opts.axon) core = None if opts.cortex: core = await s_telepath.openurl(opts.cortex) tags = set() if opts.tags: for tag in opts.tags.split(','): tags.add(tag) tags = tuple(tags) if tags: outp.printf(f'adding tags: {tags}') filepaths = set() for item in opts.filenames: paths = glob.glob(item, recursive=opts.recursive) if not paths: outp.printf(f'filepath does not contain any files: {item}') continue filepaths.update([path for path in paths if os.path.isfile(path)]) for path in filepaths: bname = os.path.basename(path) hset = s_hashset.HashSet() with s_common.reqfile(path) as fd: hset.eatfd(fd) fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes} sha256 = fhashes.get('sha256') bsha256 = s_common.uhex(sha256) if not await axon.has(bsha256): async with await axon.upload() as upfd: with s_common.genfile(path) as fd: for byts in s_common.iterfd(fd): await upfd.write(byts) size, hashval = await upfd.save() if hashval != bsha256: # pragma: no cover raise s_exc.SynErr(mesg='hashes do not match', ehash=s_common.ehex(hashval), ahash=hashval) outp.printf(f'Uploaded [{bname}] to axon') else: outp.printf(f'Axon already had [{bname}]') if core: opts = { 'vars': { 'md5': fhashes.get('md5'), 'sha1': fhashes.get('sha1'), 'sha256': fhashes.get('sha256'), 'size': hset.size, 'name': bname, 'tags': tags, } } q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \ '{ for $tag in $tags { [+#$tag] } }' msgs = await core.storm(q, opts=opts).list() node = [m[1] for m in msgs if m[0] == 'node'][0] iden = node[0][1] size = node[1]['props']['size'] name = node[1]['props']['name'] mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}' outp.printf(mesg) await axon.fini() if core: await core.fini() if telefini: # pragma: no cover await telefini() return 0
def set(self, iden, offs): buid = s_common.uhex(iden) byts = s_common.int64en(offs) self.lenv.put(buid, byts, db=self.db)
def gen(self, iden): bidn = s_common.uhex(iden) return SlabDict(self.slab, db=self.db, pref=bidn)
def set(self, iden, offs): buid = s_common.uhex(iden) byts = offs.to_bytes(length=8, byteorder='big') self.slab.put(buid, byts, db=self.db)