def test_lib_crypto_tnfl_break(self):
ekey = s_tinfoil.newkey()
tinh = s_tinfoil.TinFoilHat(ekey)
goodbyts = tinh.enc(b'foobar', b'hehe')
edict = s_msgpack.un(goodbyts)
# Empty values will fail to decrypt
for key in ('iv', 'data', 'asscd'):
bdict = {k: v for k, v in edict.items() if k != key}
byts = s_msgpack.en(bdict)
self.none(tinh.dec(byts))
# Tampered values will fail
bdict = {k: v for k, v in edict.items()}
bdict['iv'] = os.urandom(16)
byts = s_msgpack.en(bdict)
self.none(tinh.dec(byts))
bdict = {k: v for k, v in edict.items()}
bdict['data'] = os.urandom(16)
byts = s_msgpack.en(bdict)
self.none(tinh.dec(byts))
bdict = {k: v for k, v in edict.items()}
bdict['asscd'] = os.urandom(16)
byts = s_msgpack.en(bdict)
self.none(tinh.dec(byts))
def migrate_v0_rules(self):
'''
Remove any v0 (i.e. pre-010) rules from storage and replace them with v1 rules.
Notes:
v0 had two differences user was a username. Replaced with iden of user as 'iden' field.
Also 'iden' was storage as binary. Now it is stored as hex string.
'''
for iden, valu in self.core.slab.scanByFull(db=self.trigdb):
ruledict = s_msgpack.un(valu)
ver = ruledict.get('ver')
if ver != 0:
continue
user = ruledict.pop('user')
if user is None:
logger.warning('Username missing in stored trigger rule %r', iden)
continue
# In v0, stored user was username, in >0 user is useriden
user = self.core.auth.getUserByName(user).iden
if user is None:
logger.warning('Unrecognized username in stored trigger rule %r', iden)
continue
ruledict['ver'] = 1
ruledict['useriden'] = user
newiden = s_common.ehex(iden)
self.core.slab.pop(iden, db=self.trigdb)
self.core.slab.put(newiden.encode(), s_msgpack.en(ruledict), db=self.trigdb)
def decrypt(self, ciphertext):
'''
Decrypt a message, validating its sequence number is as we expect.
Args:
ciphertext (bytes): The message to decrypt and verify.
Returns:
mesg: A mesg.
Raises:
s_exc.CryptoErr: If the message decryption fails or the sequence number was unexpected.
'''
plaintext = self._rx_tinh.dec(ciphertext)
if plaintext is None:
logger.error('Message decryption failure')
raise s_exc.CryptoErr(mesg='Message decryption failure')
seqn = next(self._rx_sn)
sn, mesg = s_msgpack.un(plaintext)
if sn != seqn:
logger.error('Message out of sequence: got %d expected %d', sn, seqn)
raise s_exc.CryptoErr(mesg='Message out of sequence', expected=seqn, got=sn)
return mesg
def test_msgpack_large_data(self):
big_string = s_const.mebibyte * 129 * 'V'
struct = ('test', {'key': big_string})
buf = s_msgpack.en(struct)
unpacked_struct = s_msgpack.un(buf)
self.eq(struct, unpacked_struct)
# Ensure our use of msgpack.Unpacker can also handle this data
with self.getTestDir() as dirn:
with s_common.genfile(dirn, 'test.mpk') as fd:
fd.write(buf)
with s_common.genfile(dirn, 'test.mpk') as fd:
genr = s_msgpack.iterfd(fd)
objs = list(genr)
self.len(1, objs)
self.eq(objs[0], struct)
# Ensure that our streaming Unpk object can also handle this data
unpk = s_msgpack.Unpk()
objs = unpk.feed(buf)
self.len(1, objs)
self.eq(objs[0], (135266320, struct))
def _storPropDel(self, oper):
_, (buid, form, prop, info) = oper
fenc = form.encode() + b'\x00'
penc = prop.encode() + b'\x00'
if prop:
bpkey = buid + prop.encode()
else:
bpkey = buid + b'*' + form.encode()
univ = info.get('univ')
byts = self.layrslab.pop(bpkey, db=self.bybuid)
if byts is None:
return
del self.buidcache[buid]
oldv, oldi = s_msgpack.un(byts)
pvvalu = s_msgpack.en((buid,))
if oldi is not None:
self.layrslab.delete(fenc + penc + oldi, pvvalu, db=self.byprop)
if univ:
self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv)
def test_msgpack_types(self):
# This is a future-proofing test for msgpack to ensure that
buf = b'\x92\xa4hehe\x85\xa3str\xa41234\xa3int\xcd\x04\xd2\xa5float\xcb@(\xae\x14z\xe1G\xae\xa3bin\xc4\x041234\xa9realworld\xac\xc7\x8b\xef\xbf\xbd\xed\xa1\x82\xef\xbf\xbd\x12'
struct = (
'hehe',
{
'str': '1234',
'int': 1234,
'float': 12.34,
'bin': b'1234',
'realworld': '\u01cb\ufffd\ud842\ufffd\u0012'
}
)
unode = s_msgpack.un(buf)
self.eq(unode, struct)
# Ensure our use of msgpack.Unpacker can also handle this data
with self.getTestDir() as dirn:
with s_common.genfile(dirn, 'test.mpk') as fd:
fd.write(buf)
with s_common.genfile(dirn, 'test.mpk') as fd:
genr = s_msgpack.iterfd(fd)
objs = list(genr)
self.len(1, objs)
self.eq(objs[0], struct)
# Ensure that our streaming Unpk object can also handle this data
unpk = s_msgpack.Unpk()
objs = unpk.feed(buf)
self.len(1, objs)
self.eq(objs[0], (71, struct))
def test_msgpack_surrogates(self):
bads = '\u01cb\ufffd\ud842\ufffd\u0012'
obyts = s_msgpack.en(bads)
self.isinstance(obyts, bytes)
outs = s_msgpack.un(obyts)
self.eq(outs, bads)
with self.getTestDir() as fdir:
fd = s_common.genfile(fdir, 'test.mpk')
fd.write(obyts)
fd.close()
fd = s_common.genfile(fdir, 'test.mpk')
gen = s_msgpack.iterfd(fd)
items = [obj for obj in gen]
self.len(1, items)
self.eq(outs, bads)
fd.close()
unpk = s_msgpack.Unpk()
ret = unpk.feed(obyts)
self.len(1, ret)
self.eq([(13, bads)], ret)
async def _storLoadHive(self):
for lkey, lval in self.slab.scanByFull(db=self.db):
path = tuple(lkey.decode('utf8').split('\x00'))
valu = s_msgpack.un(lval)
await self._loadNodeValu(path, valu)
async def iterUnivRows(self, prop):
'''
Iterate (buid, valu) rows for the given universal prop
'''
penc = prop.encode()
pref = penc + b'\x00'
for _, pval in self.layrslab.scanByPref(pref, db=self.byuniv):
buid = s_msgpack.un(pval)[0]
byts = self.layrslab.get(buid + penc, db=self.bybuid)
if byts is None:
continue
valu, indx = s_msgpack.un(byts)
yield buid, valu
def getProvStack(self, iden: bytes):
'''
Returns the provenance stack given the iden to it
'''
retn = self.slab.get(iden, db=self.db)
if retn is None:
return None
return s_msgpack.un(retn)
async def iterPropIndx(self, form, prop, indx):
'''
Yield (buid, valu) tuples for the given prop with the specified indx valu
'''
penc = prop.encode()
pref = form.encode() + b'\x00' + penc + b'\x00' + indx
for _, pval in self.layrslab.scanByPref(pref, db=self.byprop):
buid = s_msgpack.un(pval)[0]
byts = self.layrslab.get(buid + penc, db=self.bybuid)
if byts is None:
continue
valu, indx = s_msgpack.un(byts)
yield buid, valu
def carve(self, tick, tock=None):
lmax = None
lmin = tick.to_bytes(8, 'big')
if tock is not None:
lmax = tock.to_bytes(8, 'big')
for lkey, byts in self.slab.scanByRange(lmin, lmax=lmax, db=self.db):
tick = int.from_bytes(lkey, 'big')
yield tick, s_msgpack.un(byts)
def _getPrefProps(self, bidn):
size = len(bidn)
props = {}
for lkey, lval in self.slab.scanByPref(bidn, db=self.db):
name = lkey[size:].decode('utf8')
props[name] = s_msgpack.un(lval)
return props
async def iterFormRows(self, form):
'''
Iterate (buid, valu) rows for the given form in this layer.
'''
# <form> 00 00 (no prop...)
pref = form.encode() + b'\x00\x00'
penc = b'*' + form.encode()
for _, pval in self.layrslab.scanByPref(pref, db=self.byprop):
buid = s_msgpack.un(pval)[0]
byts = self.layrslab.get(buid + penc, db=self.bybuid)
if byts is None:
continue
valu, indx = s_msgpack.un(byts)
yield buid, valu
async def iterPropRows(self, form, prop):
'''
Iterate (buid, valu) rows for the given form:prop in this layer.
'''
# iterate byprop and join bybuid to get to value
penc = prop.encode()
pref = form.encode() + b'\x00' + penc + b'\x00'
for _, pval in self.layrslab.scanByPref(pref, db=self.byprop):
buid = s_msgpack.un(pval)[0]
byts = self.layrslab.get(buid + penc, db=self.bybuid)
if byts is None:
continue
valu, indx = s_msgpack.un(byts)
yield buid, valu
def _load_all(self):
self.migrate_v0_rules()
for iden, valu in self.core.slab.scanByFull(db=self.trigdb):
try:
ruledict = s_msgpack.un(valu)
ver = ruledict.pop('ver')
cond = ruledict.pop('cond')
user = ruledict.pop('useriden')
query = ruledict.pop('storm')
self._load_rule(iden.decode(), ver, cond, user, query, info=ruledict)
except (KeyError, s_exc.SynErr) as e:
logger.warning('Invalid rule %r found in storage: %r', iden, e)
continue
def get(name, defval=None):
'''
Return an object from the embedded synapse data folder.
Example:
for tld in syanpse.data.get('iana.tlds'):
dostuff(tld)
NOTE: Files are named synapse/data/<name>.mpk
'''
with s_datfile.openDatFile('synapse.data/%s.mpk' % name) as fd:
return s_msgpack.un(fd.read())
async def getBuidProps(self, buid):
props = self.buidcache.get(buid, {})
if props:
return props
for lkey, lval in self.layrslab.scanByPref(buid, db=self.bybuid):
prop = lkey[32:].decode('utf8')
valu, indx = s_msgpack.un(lval)
props[prop] = valu
self.buidcache[buid] = props
return props
async def getFormTodo(self, name):
'''
Produce a deconflicted list of form values across layers
as a *copy* to avoid iter vs edit issues in the indexes.
'''
size = 0
logger.warning(f'MIGRATION: calculating form todo: {name}')
async with self.getTempSlab() as slab:
for layr in self.layers:
async for buid, valu in layr.iterFormRows(name):
slab.put(buid, s_msgpack.en(valu), overwrite=False)
size += 1
logger.warning(f'MIGRATION: {name} todo size: {size}')
for buid, byts in slab.scanByFull():
yield buid, s_msgpack.un(byts)
def _storPropSetCommon(self, buid, penc, bpkey, pvpref, univ, valu, indx):
bpval = s_msgpack.en((valu, indx))
pvvalu = s_msgpack.en((buid,))
byts = self.layrslab.replace(bpkey, bpval, db=self.bybuid)
if byts is not None:
oldv, oldi = s_msgpack.un(byts)
if oldi is not None:
self.layrslab.delete(pvpref + oldi, pvvalu, db=self.byprop)
if univ:
self.layrslab.delete(penc + oldi, pvvalu, db=self.byuniv)
if indx is not None:
self.layrslab.put(pvpref + indx, pvvalu, dupdata=True, db=self.byprop)
if univ:
self.layrslab.put(penc + indx, pvvalu, dupdata=True, db=self.byuniv)
def dec(self, byts):
'''
Decode an envelope dict and decrypt the given bytes.
Args:
byts (bytes): Bytes to decrypt.
Returns:
bytes: Decrypted message.
'''
envl = s_msgpack.un(byts)
iv = envl.get('iv', b'')
asscd = envl.get('asscd', b'')
data = envl.get('data', b'')
decryptor = AESGCM(self.ekey)
try:
data = decryptor.decrypt(iv, data, asscd)
except Exception:
logger.exception('Error decrypting data')
return None
return data
async def _storBuidSet(self, oper):
'''
Migration-only method
Notes:
Precondition: buid cache must be disabled
'''
assert self.buidcache.disabled
_, (form, oldb, newb) = oper
fenc = form.encode() + b'\x00'
pvoldval = s_msgpack.en((oldb,))
pvnewval = s_msgpack.en((newb,))
for lkey, lval in self.layrslab.scanByPref(oldb, db=self.bybuid):
proputf8 = lkey[32:]
valu, indx = s_msgpack.un(lval)
if indx is not None:
# <prop><00><indx>
propindx = proputf8 + b'\x00' + indx
if proputf8[0] in (46, 35): # ".univ" or "#tag"
self.layrslab.put(propindx, pvnewval, dupdata=True, db=self.byuniv)
self.layrslab.delete(propindx, pvoldval, db=self.byuniv)
bypropkey = fenc + propindx
self.layrslab.put(bypropkey, pvnewval, db=self.byprop)
self.layrslab.delete(bypropkey, pvoldval, db=self.byprop)
self.layrslab.put(newb + proputf8, lval, db=self.bybuid)
self.layrslab.delete(lkey, db=self.bybuid)
def getDocData(fp, root=None):
'''
Args:
fn (str): Name of the file to retrieve the data of.
root (str): Optional root path to look for a docdata directory in.
Notes:
Will detect json/jsonl/yaml/mpk extensions and automatically
decode that data if found; otherwise it returns bytes.
Defaults to looking for the ``docdata`` directory in the current
working directory. This behavior works fine for notebooks nested
in the docs directory of synapse; but this root directory that
is looked for may be overridden by providing an alternative root.
Returns:
data: May be deserialized data or bytes.
Raises:
ValueError if the file does not exist or directory traversal attempted..
'''
fpath = getDocPath(fp, root)
if fpath.endswith('.yaml'):
return s_common.yamlload(fpath)
if fpath.endswith('.json'):
return s_common.jsload(fpath)
with s_common.genfile(fpath) as fd:
if fpath.endswith('.mpk'):
return s_msgpack.un(fd.read())
if fpath.endswith('.jsonl'):
recs = []
for line in fd.readlines():
recs.append(json.loads(line.decode()))
return recs
return fd.read()
async def cellAuthToHive(dirn, auth):
'''
Migrate old cell Auth() data into a HiveAuth().
'''
logger.warning('migrating old cell auth to hive')
path = os.path.join(dirn, 'auth.lmdb')
lenv = lmdb.open(path, max_dbs=128)
userdb = lenv.open_db(b'users')
roledb = lenv.open_db(b'roles')
migrated_roles = False
migrated_users = False
with lenv.begin() as xact:
with xact.cursor(db=roledb) as curs:
for lkey, lval in curs.iternext():
name = lkey.decode('utf8')
info = s_msgpack.un(lval)
logger.info(f'Migrating role: {name}')
role = auth.getRoleByName(name)
if role is None:
logger.info(f'Creating role: {name}')
role = await auth.addRole(name)
rules = info.get('rules', ())
await role.setRules(rules)
migrated_roles = True
if not migrated_roles: # pragma: no cover
logger.info('No roles were migrated.')
with xact.cursor(db=userdb) as curs:
for lkey, lval in curs.iternext():
name = lkey.decode('utf8')
info = s_msgpack.un(lval)
logger.info(f'Migrating user: {name}')
user = auth.getUserByName(name)
if user is None:
logger.info(f'Creating user: {name}')
user = await auth.addUser(name)
if info.get('admin', False):
await user.setAdmin(True)
if info.get('locked', False):
await user.setLocked(True)
# set this directly since we only have the shadow
shadow = info.get('shadow')
if shadow is not None:
await user.info.set('passwd', shadow)
rules = info.get('rules', ())
await user.setRules(rules)
for name in info.get('roles', ()):
await user.grant(name)
migrated_users = True
if not migrated_users: # pragma: no cover
logger.info('No users were migrated.')
lenv.sync()
lenv.close()
async def getFormIndx(self, buid):
for lkey, lval in self.layrslab.scanByPref(buid + b'*', db=self.bybuid):
valu, indx = s_msgpack.un(lval)
return indx
async def editNodeNdef(self, oldv, newv):
'''
Migration-only method
Notes:
Precondition: buid cache must be disabled
'''
assert self.buidcache.disabled
oldb = s_common.buid(oldv)
newb = s_common.buid(newv)
pvoldval = s_msgpack.en((oldb,))
pvnewval = s_msgpack.en((newb,))
oldfenc = oldv[0].encode() + b'\x00'
newfenc = newv[0].encode() + b'\x00'
newprel = b'*' + newv[0].encode()
newnindx = self.core.model.prop(newv[0]).type.indx(newv[1])
# avoid any potential iter/edit issues...
todo = list(self.layrslab.scanByPref(oldb, db=self.bybuid))
for lkey, lval in todo:
proputf8 = lkey[32:]
valu, indx = s_msgpack.un(lval)
# for the *<form> prop, the byprop index has <form><00><00><indx>
if proputf8[0] == 42:
newpropkey = newfenc + b'\x00' + newnindx
if indx is not None:
oldpropkey = oldfenc + b'\x00' + indx
if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover
logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}')
self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop)
byts = s_msgpack.en((newv[1], newnindx))
self.layrslab.put(newb + newprel, byts, db=self.bybuid)
else:
# <prop><00><indx>
propindx = proputf8 + b'\x00' + indx
if proputf8[0] in (46, 35): # ".univ" or "#tag"
self.layrslab.put(propindx, pvnewval, dupdata=True, db=self.byuniv)
self.layrslab.delete(propindx, pvoldval, db=self.byuniv)
oldpropkey = oldfenc + propindx
newpropkey = newfenc + propindx
if not self.layrslab.delete(oldpropkey, pvoldval, db=self.byprop): # pragma: no cover
logger.warning(f'editNodeNdef del byprop missing for {repr(oldv)} {repr(oldpropkey)}')
self.layrslab.put(newpropkey, pvnewval, dupdata=True, db=self.byprop)
self.layrslab.put(newb + proputf8, lval, db=self.bybuid)
self.layrslab.delete(lkey, db=self.bybuid)
def _rowsByEq(self, db, pref, valu):
lkey = pref + valu
for _, byts in self.layrslab.scanByDups(lkey, db=db):
yield s_msgpack.un(byts)
def _rowsByPref(self, db, pref, valu):
pref = pref + valu
for _, byts in self.layrslab.scanByPref(pref, db=db):
yield s_msgpack.un(byts)
def _rowsByRange(self, db, pref, valu):
lmin = pref + valu[0]
lmax = pref + valu[1]
for _, byts in self.layrslab.scanByRange(lmin, lmax, db=db):
yield s_msgpack.un(byts)
def test_lib_crypto_tnfl_base(self):
ekey = s_tinfoil.newkey()
self.len(32, ekey)
self.isinstance(ekey, bytes)
# Keys are random from s_tinfoil.newkey
self.ne(ekey, s_tinfoil.newkey())
self.ne(ekey, s_tinfoil.newkey())
tinh = s_tinfoil.TinFoilHat(ekey)
self.true(tinh.bend is default_backend())
byts = tinh.enc(b'foobar')
# Ensure the envelope is shaped as we expect it too be
edict = s_msgpack.un(byts)
self.isinstance(edict, dict)
self.len(3, edict)
data = edict.get('data')
self.isinstance(data, bytes)
self.len(6 + 16, data)
iv = edict.get('iv')
self.isinstance(iv, bytes)
self.len(16, iv)
asscd = edict.get('asscd')
self.eq(asscd, None)
# We can decrypt and get our original message back
self.eq(tinh.dec(byts), b'foobar')
# There isn't anythign special about the tinfoilhat object
# We can make a new one to decrypt our existing message with
# the known key
self.eq(s_tinfoil.TinFoilHat(ekey).dec(byts), b'foobar')
# We can encrypt/decrypt null messages
byts = tinh.enc(b'')
self.eq(tinh.dec(byts), b'')
# Attempting to decrypt with the wrong key fails
self.none(s_tinfoil.TinFoilHat(s_tinfoil.newkey()).dec(byts))
# Messages are stream encoded so the length is 1 to 1
for msize in [0, 1, 2, 15, 16, 17, 31, 32, 33, 63, 65]:
mesg = msize * b'!'
byts = tinh.enc(mesg)
edict = s_msgpack.un(byts)
self.len(16, edict.get('iv'))
data = edict.get('data')
self.len(len(mesg) + 16, data)
self.eq(tinh.dec(byts), mesg)
# We can pass in additional data that we want authed too
byts = tinh.enc(b'robert grey', b'pennywise')
edict = s_msgpack.un(byts)
self.eq(edict.get('asscd'), b'pennywise')
self.eq(tinh.dec(byts), b'robert grey')
# A malformed edict with a bad asscd won't decrypt
edict['asscd'] = b'georgey'
self.none(tinh.dec(s_msgpack.en(edict)))
