def test_lib_crypto_tnfl_break(self): ekey = s_tinfoil.newkey() tinh = s_tinfoil.TinFoilHat(ekey) goodbyts = tinh.enc(b'foobar', b'hehe') edict = s_msgpack.un(goodbyts) # Empty values will fail to decrypt for key in ('iv', 'data', 'asscd'): bdict = {k: v for k, v in edict.items() if k != key} byts = s_msgpack.en(bdict) self.none(tinh.dec(byts)) # Tampered values will fail bdict = {k: v for k, v in edict.items()} bdict['iv'] = os.urandom(16) byts = s_msgpack.en(bdict) self.none(tinh.dec(byts)) bdict = {k: v for k, v in edict.items()} bdict['data'] = os.urandom(16) byts = s_msgpack.en(bdict) self.none(tinh.dec(byts)) bdict = {k: v for k, v in edict.items()} bdict['asscd'] = os.urandom(16) byts = s_msgpack.en(bdict) self.none(tinh.dec(byts))
def migrate_v0_rules(self): ''' Remove any v0 (i.e. pre-010) rules from storage and replace them with v1 rules. Notes: v0 had two differences user was a username. Replaced with iden of user as 'iden' field. Also 'iden' was storage as binary. Now it is stored as hex string. ''' for iden, valu in self.core.slab.scanByFull(db=self.trigdb): ruledict = s_msgpack.un(valu) ver = ruledict.get('ver') if ver != 0: continue user = ruledict.pop('user') if user is None: logger.warning('Username missing in stored trigger rule %r', iden) continue # In v0, stored user was username, in >0 user is useriden user = self.core.auth.getUserByName(user).iden if user is None: logger.warning('Unrecognized username in stored trigger rule %r', iden) continue ruledict['ver'] = 1 ruledict['useriden'] = user newiden = s_common.ehex(iden) self.core.slab.pop(iden, db=self.trigdb) self.core.slab.put(newiden.encode(), s_msgpack.en(ruledict), db=self.trigdb)
def decrypt(self, ciphertext): ''' Decrypt a message, validating its sequence number is as we expect. Args: ciphertext (bytes): The message to decrypt and verify. Returns: mesg: A mesg. Raises: s_exc.CryptoErr: If the message decryption fails or the sequence number was unexpected. ''' plaintext = self._rx_tinh.dec(ciphertext) if plaintext is None: logger.error('Message decryption failure') raise s_exc.CryptoErr(mesg='Message decryption failure') seqn = next(self._rx_sn) sn, mesg = s_msgpack.un(plaintext) if sn != seqn: logger.error('Message out of sequence: got %d expected %d', sn, seqn) raise s_exc.CryptoErr(mesg='Message out of sequence', expected=seqn, got=sn) return mesg
def test_msgpack_large_data(self): big_string = s_const.mebibyte * 129 * 'V' struct = ('test', {'key': big_string}) buf = s_msgpack.en(struct) unpacked_struct = s_msgpack.un(buf) self.eq(struct, unpacked_struct) # Ensure our use of msgpack.Unpacker can also handle this data with self.getTestDir() as dirn: with s_common.genfile(dirn, 'test.mpk') as fd: fd.write(buf) with s_common.genfile(dirn, 'test.mpk') as fd: genr = s_msgpack.iterfd(fd) objs = list(genr) self.len(1, objs) self.eq(objs[0], struct) # Ensure that our streaming Unpk object can also handle this data unpk = s_msgpack.Unpk() objs = unpk.feed(buf) self.len(1, objs) self.eq(objs[0], (135266320, struct))
def _storPropDel(self, oper): _, (buid, form, prop, info) = oper fenc = form.encode() + b'\x00' penc = prop.encode() + b'\x00' if prop: bpkey = buid + prop.encode() else: bpkey = buid + b'*' + form.encode() univ = info.get('univ') byts = self.layrslab.pop(bpkey, db=self.bybuid) if byts is None: return del self.buidcache[buid] oldv, oldi = s_msgpack.un(byts) pvvalu = s_msgpack.en((buid,)) if oldi is not None: self.layrslab.delete(fenc + penc + oldi, pvvalu, db=self.byprop) if univ: self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv)
def test_msgpack_types(self): # This is a future-proofing test for msgpack to ensure that buf = b'\x92\xa4hehe\x85\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12' struct = ( 'hehe', { 'str': '1234', 'int': 1234, 'float': 12.34, 'bin': b'1234', 'realworld': '\u01cb\ufffd\ud842\ufffd\u0012' } ) unode = s_msgpack.un(buf) self.eq(unode, struct) # Ensure our use of msgpack.Unpacker can also handle this data with self.getTestDir() as dirn: with s_common.genfile(dirn, 'test.mpk') as fd: fd.write(buf) with s_common.genfile(dirn, 'test.mpk') as fd: genr = s_msgpack.iterfd(fd) objs = list(genr) self.len(1, objs) self.eq(objs[0], struct) # Ensure that our streaming Unpk object can also handle this data unpk = s_msgpack.Unpk() objs = unpk.feed(buf) self.len(1, objs) self.eq(objs[0], (71, struct))
def test_msgpack_surrogates(self): bads = '\u01cb\ufffd\ud842\ufffd\u0012' obyts = s_msgpack.en(bads) self.isinstance(obyts, bytes) outs = s_msgpack.un(obyts) self.eq(outs, bads) with self.getTestDir() as fdir: fd = s_common.genfile(fdir, 'test.mpk') fd.write(obyts) fd.close() fd = s_common.genfile(fdir, 'test.mpk') gen = s_msgpack.iterfd(fd) items = [obj for obj in gen] self.len(1, items) self.eq(outs, bads) fd.close() unpk = s_msgpack.Unpk() ret = unpk.feed(obyts) self.len(1, ret) self.eq([(13, bads)], ret)
async def _storLoadHive(self): for lkey, lval in self.slab.scanByFull(db=self.db): path = tuple(lkey.decode('utf8').split('\x00')) valu = s_msgpack.un(lval) await self._loadNodeValu(path, valu)
async def iterUnivRows(self, prop): ''' Iterate (buid, valu) rows for the given universal prop ''' penc = prop.encode() pref = penc + b'\x00' for _, pval in self.layrslab.scanByPref(pref, db=self.byuniv): buid = s_msgpack.un(pval)[0] byts = self.layrslab.get(buid + penc, db=self.bybuid) if byts is None: continue valu, indx = s_msgpack.un(byts) yield buid, valu
def getProvStack(self, iden: bytes): ''' Returns the provenance stack given the iden to it ''' retn = self.slab.get(iden, db=self.db) if retn is None: return None return s_msgpack.un(retn)
async def iterPropIndx(self, form, prop, indx): ''' Yield (buid, valu) tuples for the given prop with the specified indx valu ''' penc = prop.encode() pref = form.encode() + b'\x00' + penc + b'\x00' + indx for _, pval in self.layrslab.scanByPref(pref, db=self.byprop): buid = s_msgpack.un(pval)[0] byts = self.layrslab.get(buid + penc, db=self.bybuid) if byts is None: continue valu, indx = s_msgpack.un(byts) yield buid, valu
def carve(self, tick, tock=None): lmax = None lmin = tick.to_bytes(8, 'big') if tock is not None: lmax = tock.to_bytes(8, 'big') for lkey, byts in self.slab.scanByRange(lmin, lmax=lmax, db=self.db): tick = int.from_bytes(lkey, 'big') yield tick, s_msgpack.un(byts)
def _getPrefProps(self, bidn): size = len(bidn) props = {} for lkey, lval in self.slab.scanByPref(bidn, db=self.db): name = lkey[size:].decode('utf8') props[name] = s_msgpack.un(lval) return props
async def iterFormRows(self, form): ''' Iterate (buid, valu) rows for the given form in this layer. ''' # <form> 00 00 (no prop...) pref = form.encode() + b'\x00\x00' penc = b'*' + form.encode() for _, pval in self.layrslab.scanByPref(pref, db=self.byprop): buid = s_msgpack.un(pval)[0] byts = self.layrslab.get(buid + penc, db=self.bybuid) if byts is None: continue valu, indx = s_msgpack.un(byts) yield buid, valu
async def iterPropRows(self, form, prop): ''' Iterate (buid, valu) rows for the given form:prop in this layer. ''' # iterate byprop and join bybuid to get to value penc = prop.encode() pref = form.encode() + b'\x00' + penc + b'\x00' for _, pval in self.layrslab.scanByPref(pref, db=self.byprop): buid = s_msgpack.un(pval)[0] byts = self.layrslab.get(buid + penc, db=self.bybuid) if byts is None: continue valu, indx = s_msgpack.un(byts) yield buid, valu
def _load_all(self): self.migrate_v0_rules() for iden, valu in self.core.slab.scanByFull(db=self.trigdb): try: ruledict = s_msgpack.un(valu) ver = ruledict.pop('ver') cond = ruledict.pop('cond') user = ruledict.pop('useriden') query = ruledict.pop('storm') self._load_rule(iden.decode(), ver, cond, user, query, info=ruledict) except (KeyError, s_exc.SynErr) as e: logger.warning('Invalid rule %r found in storage: %r', iden, e) continue
def get(name, defval=None): ''' Return an object from the embedded synapse data folder. Example: for tld in syanpse.data.get('iana.tlds'): dostuff(tld) NOTE: Files are named synapse/data/<name>.mpk ''' with s_datfile.openDatFile('synapse.data/%s.mpk' % name) as fd: return s_msgpack.un(fd.read())
async def getBuidProps(self, buid): props = self.buidcache.get(buid, {}) if props: return props for lkey, lval in self.layrslab.scanByPref(buid, db=self.bybuid): prop = lkey[32:].decode('utf8') valu, indx = s_msgpack.un(lval) props[prop] = valu self.buidcache[buid] = props return props
async def getFormTodo(self, name): ''' Produce a deconflicted list of form values across layers as a *copy* to avoid iter vs edit issues in the indexes. ''' size = 0 logger.warning(f'MIGRATION: calculating form todo: {name}') async with self.getTempSlab() as slab: for layr in self.layers: async for buid, valu in layr.iterFormRows(name): slab.put(buid, s_msgpack.en(valu), overwrite=False) size += 1 logger.warning(f'MIGRATION: {name} todo size: {size}') for buid, byts in slab.scanByFull(): yield buid, s_msgpack.un(byts)
def _storPropSetCommon(self, buid, penc, bpkey, pvpref, univ, valu, indx): bpval = s_msgpack.en((valu, indx)) pvvalu = s_msgpack.en((buid,)) byts = self.layrslab.replace(bpkey, bpval, db=self.bybuid) if byts is not None: oldv, oldi = s_msgpack.un(byts) if oldi is not None: self.layrslab.delete(pvpref + oldi, pvvalu, db=self.byprop) if univ: self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv) if indx is not None: self.layrslab.put(pvpref + indx, pvvalu, dupdata=True, db=self.byprop) if univ: self.layrslab.put(penc + indx, pvvalu, dupdata=True, db=self.byuniv)
def dec(self, byts): ''' Decode an envelope dict and decrypt the given bytes. Args: byts (bytes): Bytes to decrypt. Returns: bytes: Decrypted message. ''' envl = s_msgpack.un(byts) iv = envl.get('iv', b'') asscd = envl.get('asscd', b'') data = envl.get('data', b'') decryptor = AESGCM(self.ekey) try: data = decryptor.decrypt(iv, data, asscd) except Exception: logger.exception('Error decrypting data') return None return data
async def _storBuidSet(self, oper): ''' Migration-only method Notes: Precondition: buid cache must be disabled ''' assert self.buidcache.disabled _, (form, oldb, newb) = oper fenc = form.encode() + b'\x00' pvoldval = s_msgpack.en((oldb,)) pvnewval = s_msgpack.en((newb,)) for lkey, lval in self.layrslab.scanByPref(oldb, db=self.bybuid): proputf8 = lkey[32:] valu, indx = s_msgpack.un(lval) if indx is not None: # <prop><00><indx> propindx = proputf8 + b'\x00' + indx if proputf8[0] in (46, 35): # ".univ" or "#tag" self.layrslab.put(propindx, pvnewval, dupdata=True, db=self.byuniv) self.layrslab.delete(propindx, pvoldval, db=self.byuniv) bypropkey = fenc + propindx self.layrslab.put(bypropkey, pvnewval, db=self.byprop) self.layrslab.delete(bypropkey, pvoldval, db=self.byprop) self.layrslab.put(newb + proputf8, lval, db=self.bybuid) self.layrslab.delete(lkey, db=self.bybuid)
def getDocData(fp, root=None): ''' Args: fn (str): Name of the file to retrieve the data of. root (str): Optional root path to look for a docdata directory in. Notes: Will detect json/jsonl/yaml/mpk extensions and automatically decode that data if found; otherwise it returns bytes. Defaults to looking for the ``docdata`` directory in the current working directory. This behavior works fine for notebooks nested in the docs directory of synapse; but this root directory that is looked for may be overridden by providing an alternative root. Returns: data: May be deserialized data or bytes. Raises: ValueError if the file does not exist or directory traversal attempted.. ''' fpath = getDocPath(fp, root) if fpath.endswith('.yaml'): return s_common.yamlload(fpath) if fpath.endswith('.json'): return s_common.jsload(fpath) with s_common.genfile(fpath) as fd: if fpath.endswith('.mpk'): return s_msgpack.un(fd.read()) if fpath.endswith('.jsonl'): recs = [] for line in fd.readlines(): recs.append(json.loads(line.decode())) return recs return fd.read()
async def cellAuthToHive(dirn, auth): ''' Migrate old cell Auth() data into a HiveAuth(). ''' logger.warning('migrating old cell auth to hive') path = os.path.join(dirn, 'auth.lmdb') lenv = lmdb.open(path, max_dbs=128) userdb = lenv.open_db(b'users') roledb = lenv.open_db(b'roles') migrated_roles = False migrated_users = False with lenv.begin() as xact: with xact.cursor(db=roledb) as curs: for lkey, lval in curs.iternext(): name = lkey.decode('utf8') info = s_msgpack.un(lval) logger.info(f'Migrating role: {name}') role = auth.getRoleByName(name) if role is None: logger.info(f'Creating role: {name}') role = await auth.addRole(name) rules = info.get('rules', ()) await role.setRules(rules) migrated_roles = True if not migrated_roles: # pragma: no cover logger.info('No roles were migrated.') with xact.cursor(db=userdb) as curs: for lkey, lval in curs.iternext(): name = lkey.decode('utf8') info = s_msgpack.un(lval) logger.info(f'Migrating user: {name}') user = auth.getUserByName(name) if user is None: logger.info(f'Creating user: {name}') user = await auth.addUser(name) if info.get('admin', False): await user.setAdmin(True) if info.get('locked', False): await user.setLocked(True) # set this directly since we only have the shadow shadow = info.get('shadow') if shadow is not None: await user.info.set('passwd', shadow) rules = info.get('rules', ()) await user.setRules(rules) for name in info.get('roles', ()): await user.grant(name) migrated_users = True if not migrated_users: # pragma: no cover logger.info('No users were migrated.') lenv.sync() lenv.close()
async def getFormIndx(self, buid): for lkey, lval in self.layrslab.scanByPref(buid + b'*', db=self.bybuid): valu, indx = s_msgpack.un(lval) return indx
async def editNodeNdef(self, oldv, newv): ''' Migration-only method Notes: Precondition: buid cache must be disabled ''' assert self.buidcache.disabled oldb = s_common.buid(oldv) newb = s_common.buid(newv) pvoldval = s_msgpack.en((oldb,)) pvnewval = s_msgpack.en((newb,)) oldfenc = oldv[0].encode() + b'\x00' newfenc = newv[0].encode() + b'\x00' newprel = b'*' + newv[0].encode() newnindx = self.core.model.prop(newv[0]).type.indx(newv[1]) # avoid any potential iter/edit issues... todo = list(self.layrslab.scanByPref(oldb, db=self.bybuid)) for lkey, lval in todo: proputf8 = lkey[32:] valu, indx = s_msgpack.un(lval) # for the *<form> prop, the byprop index has <form><00><00><indx> if proputf8[0] == 42: newpropkey = newfenc + b'\x00' + newnindx if indx is not None: oldpropkey = oldfenc + b'\x00' + indx if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}') self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop) byts = s_msgpack.en((newv[1], newnindx)) self.layrslab.put(newb + newprel, byts, db=self.bybuid) else: # <prop><00><indx> propindx = proputf8 + b'\x00' + indx if proputf8[0] in (46, 35): # ".univ" or "#tag" self.layrslab.put(propindx, pvnewval, dupdata=True, db=self.byuniv) self.layrslab.delete(propindx, pvoldval, db=self.byuniv) oldpropkey = oldfenc + propindx newpropkey = newfenc + propindx if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}') self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop) self.layrslab.put(newb + proputf8, lval, db=self.bybuid) self.layrslab.delete(lkey, db=self.bybuid)
def _rowsByEq(self, db, pref, valu): lkey = pref + valu for _, byts in self.layrslab.scanByDups(lkey, db=db): yield s_msgpack.un(byts)
def _rowsByPref(self, db, pref, valu): pref = pref + valu for _, byts in self.layrslab.scanByPref(pref, db=db): yield s_msgpack.un(byts)
def _rowsByRange(self, db, pref, valu): lmin = pref + valu[0] lmax = pref + valu[1] for _, byts in self.layrslab.scanByRange(lmin, lmax, db=db): yield s_msgpack.un(byts)
def test_lib_crypto_tnfl_base(self): ekey = s_tinfoil.newkey() self.len(32, ekey) self.isinstance(ekey, bytes) # Keys are random from s_tinfoil.newkey self.ne(ekey, s_tinfoil.newkey()) self.ne(ekey, s_tinfoil.newkey()) tinh = s_tinfoil.TinFoilHat(ekey) self.true(tinh.bend is default_backend()) byts = tinh.enc(b'foobar') # Ensure the envelope is shaped as we expect it too be edict = s_msgpack.un(byts) self.isinstance(edict, dict) self.len(3, edict) data = edict.get('data') self.isinstance(data, bytes) self.len(6 + 16, data) iv = edict.get('iv') self.isinstance(iv, bytes) self.len(16, iv) asscd = edict.get('asscd') self.eq(asscd, None) # We can decrypt and get our original message back self.eq(tinh.dec(byts), b'foobar') # There isn't anythign special about the tinfoilhat object # We can make a new one to decrypt our existing message with # the known key self.eq(s_tinfoil.TinFoilHat(ekey).dec(byts), b'foobar') # We can encrypt/decrypt null messages byts = tinh.enc(b'') self.eq(tinh.dec(byts), b'') # Attempting to decrypt with the wrong key fails self.none(s_tinfoil.TinFoilHat(s_tinfoil.newkey()).dec(byts)) # Messages are stream encoded so the length is 1 to 1 for msize in [0, 1, 2, 15, 16, 17, 31, 32, 33, 63, 65]: mesg = msize * b'!' byts = tinh.enc(mesg) edict = s_msgpack.un(byts) self.len(16, edict.get('iv')) data = edict.get('data') self.len(len(mesg) + 16, data) self.eq(tinh.dec(byts), mesg) # We can pass in additional data that we want authed too byts = tinh.enc(b'robert grey', b'pennywise') edict = s_msgpack.un(byts) self.eq(edict.get('asscd'), b'pennywise') self.eq(tinh.dec(byts), b'robert grey') # A malformed edict with a bad asscd won't decrypt edict['asscd'] = b'georgey' self.none(tinh.dec(s_msgpack.en(edict)))