async def test_server_mirror_badiden(self):
with self.getTestDir() as dirn, self.withSetLoggingMock():
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
argv0 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--name', 'srccore',
path00,
]
out0 = self.getTestOutp()
async with await s_cortex.Cortex.initFromArgv(argv0, outp=out0) as core00:
out1 = self.getTestOutp()
argv1 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--mirror', core00.getLocalUrl(),
path01,
]
s_common.yamlsave({'logchanges': True}, path01, 'cell.yaml')
async with await s_cortex.Cortex.initFromArgv(argv1, outp=out1) as core01:
await core01.waitfini(6)
self.true(core01.isfini)
async def __anit__(self, dirn, conf=None):
await s_cell.Cell.__anit__(self, dirn)
# share ourself via the cell dmon as "axon"
# for potential default remote use
self.dmon.share('axon', self)
path = s_common.gendir(self.dirn, 'axon.lmdb')
self.axonslab = await s_lmdbslab.Slab.anit(path)
self.sizes = self.axonslab.initdb('sizes')
self.onfini(self.axonslab.fini)
self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history')
path = s_common.gendir(self.dirn, 'blob.lmdb')
self.blobslab = await s_lmdbslab.Slab.anit(path)
self.blobs = self.blobslab.initdb('blobs')
self.onfini(self.blobslab.fini)
self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn')
node = await self.hive.open(('axon', 'metrics'))
self.axonmetrics = await node.dict()
self.axonmetrics.setdefault('size:bytes', 0)
self.axonmetrics.setdefault('file:count', 0)
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
async def test_server_mirror_badiden(self):
with self.getTestDir() as dirn:
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
argv0 = [
'--telepath',
'tcp://127.0.0.1:0/',
'--https',
'0',
'--name',
'srccore',
path00,
]
out0 = self.getTestOutp()
async with await s_s_cortex.main(argv0, outp=out0) as core00:
out1 = self.getTestOutp()
argv1 = [
'--telepath',
'tcp://127.0.0.1:0/',
'--https',
'0',
'--mirror',
core00.getLocalUrl(),
path01,
]
async with await s_s_cortex.main(argv1, outp=out1) as core01:
await core01.waitfini(6)
self.true(core01.isfini)
async def test_stormlib_backup(self):
with self.getTestDir() as dirn:
backdirn = s_common.gendir(dirn, 'backup')
coredirn = s_common.gendir(dirn, 'cortex')
s_common.yamlsave({'backup:dir': backdirn}, coredirn, 'cell.yaml')
async with self.getTestCore(dirn=coredirn) as core:
self.eq((), await core.callStorm('return($lib.backup.list())'))
name = await core.callStorm('return($lib.backup.run())')
self.true(os.path.isdir(os.path.join(backdirn, name)))
self.eq((name, ), await
core.callStorm('return($lib.backup.list())'))
await core.callStorm('$lib.backup.del($name)',
opts={'vars': {
'name': name
}})
self.false(os.path.isdir(os.path.join(backdirn, name)))
await core.callStorm('$lib.backup.run(name=foo)',
opts={'vars': {
'name': name
}})
self.true(os.path.isdir(os.path.join(backdirn, 'foo')))
await core.callStorm('$lib.backup.del(foo)')
self.false(os.path.isdir(os.path.join(backdirn, 'foo')))
async def test_server_mirror_restart(self):
with self.getTestDir() as dirn, self.withSetLoggingMock():
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
conf00 = {
'layer:lmdb:map_async': True,
'provenance:en': True,
'nexslog:en': True,
'layers:logedits': True,
'auth:passwd': 'secret',
}
async with await s_cortex.Cortex.anit(dirn=path00,
conf=conf00) as core00:
await core00.nodes('[ inet:asn=0 ]')
s_tools_backup.backup(path00, path01)
async with await s_cortex.Cortex.anit(dirn=path00,
conf=conf00) as core00:
# add a node for core01 to sync before window
await core00.nodes('[ inet:asn=1 ]')
argv = [
'--telepath', 'tcp://127.0.0.1:0/', '--https', '0',
'--auth-passwd', 'secret', '--mirror',
core00.getLocalUrl(), path01
]
async with await s_cortex.Cortex.initFromArgv(argv) as core01:
await core01.sync()
self.len(1, await core01.nodes('inet:asn=0'))
self.len(1, await core01.nodes('inet:asn=1'))
# get the nexus index
nexusind = core01.nexsroot.nexslog.index()
await core00.nodes('[ inet:asn=2 ]')
async with await s_cortex.Cortex.initFromArgv(argv) as core01:
# check that startup does not create any events
self.eq(nexusind, core01.nexsroot.nexslog.index())
await core01.sync()
self.len(1, await core01.nodes('inet:asn=0'))
self.len(1, await core01.nodes('inet:asn=1'))
self.len(1, await core01.nodes('inet:asn=2'))
await core00.nodes('[ inet:asn=3 ]')
await core01.sync()
self.len(4, await core01.nodes('inet:asn'))
def main(argv, outp=s_output.stdout):
argp = getArgParser()
opts = argp.parse_args(argv)
s_common.setlogging(logger, opts.log_level)
dirn = s_common.gendir(opts.dmonpath)
outp.printf(f'Beginning dmon from {dirn}')
# since we're dmon main, register our certdir as global
certdir = s_common.gendir(dirn, 'certs')
# set the default certdir to the dmon certdir
s_certdir.defdir = certdir
path = os.path.join(dirn, 'dmon.yaml')
if not os.path.isfile(path):
with open(path, 'wb') as fd:
fd.write(dmonyaml.encode('utf8'))
dmon = None
try:
dmon = s_glob.sync(s_daemon.Daemon.anit(opts.dmonpath))
host, port = dmon.addr
outp.printf(f'dmon listening on: {host}:{port}')
dmon.main()
finally:
if dmon is not None:
s_glob.sync(dmon.fini())
async def test_server_mirror_badiden(self):
with self.getTestDir() as dirn:
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
argv0 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--name', 'srccore',
path00,
]
out0 = self.getTestOutp()
async with await s_s_cortex.main(argv0, outp=out0) as core00:
out1 = self.getTestOutp()
argv1 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--mirror', core00.getLocalUrl(),
path01,
]
async with await s_s_cortex.main(argv1, outp=out1) as core01:
await core01.waitfini(6)
self.true(core01.isfini)
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
async def test_server_mirror(self):
with self.getTestDir() as dirn:
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
async with self.getTestCore(dirn=path00) as core00:
await core00.nodes('[ inet:ipv4=1.2.3.4 ]')
s_tools_backup.backup(path00, path01)
async with self.getTestCore(dirn=path00) as core00:
outp = self.getTestOutp()
argv = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--mirror', core00.getLocalUrl(),
path01]
async with await s_s_cortex.main(argv, outp=outp) as core01:
evnt = await core01._getWaitFor('inet:ipv4', '5.5.5.5')
await core00.nodes('[ inet:ipv4=5.5.5.5 ]')
await evnt.wait()
self.len(1, await core01.nodes('inet:ipv4=5.5.5.5'))
async def test_lib_aha_mirrors(self):
with self.getTestDir() as dirn:
dir0 = s_common.gendir(dirn, 'aha0')
dir1 = s_common.gendir(dirn, 'aha1')
conf = {'nexslog:en': True}
async with self.getTestAha(conf={'nexslog:en': True},
dirn=dir0) as aha0:
user = await aha0.auth.addUser('reguser', passwd='secret')
await user.setAdmin(True)
s_tools_backup.backup(dir0, dir1)
async with self.getTestAha(conf=conf, dirn=dir0) as aha0:
upstream_url = aha0.getLocalUrl()
mirrorconf = {
'nexslog:en': True,
'mirror': upstream_url,
}
async with self.getTestAha(conf=mirrorconf, dirn=dir1) as aha1:
# CA is nexus-fied
cabyts = await aha0.genCaCert('mirrorca')
await aha1.sync()
mirbyts = await aha1.genCaCert('mirrorca')
self.eq(cabyts, mirbyts)
iden = s_common.guid()
# Adding, downing, and removing service is also nexusified
info = {
'urlinfo': {
'host': '127.0.0.1',
'port': 8080,
'scheme': 'tcp'
},
'online': iden
}
await aha0.addAhaSvc('test', info, network='example.net')
await aha1.sync()
mnfo = await aha1.getAhaSvc('test.example.net')
self.eq(mnfo.get('name'), 'test.example.net')
wait00 = aha0.waiter(1, 'aha:svcdown')
await aha0.setAhaSvcDown('test',
iden,
network='example.net')
self.len(1, await wait00.wait(timeout=6))
await aha1.sync()
mnfo = await aha1.getAhaSvc('test.example.net')
self.notin('online', mnfo)
await aha0.delAhaSvc('test', network='example.net')
await aha1.sync()
mnfo = await aha1.getAhaSvc('test.example.net')
self.none(mnfo)
async def test_storm_http_inject_ca(self):
with self.getTestDir() as dirn:
cdir = s_common.gendir(dirn, 'certs')
cadir = s_common.gendir(cdir, 'cas')
tdir = s_certdir.CertDir(cdir)
tdir.genCaCert('somelocalca')
tdir.genHostCert('localhost', signas='somelocalca')
localkeyfp = tdir.getHostKeyPath('localhost')
localcertfp = tdir.getHostCertPath('localhost')
shutil.copyfile(localkeyfp, s_common.genpath(dirn, 'sslkey.pem'))
shutil.copyfile(localcertfp, s_common.genpath(dirn, 'sslcert.pem'))
tlscadir = s_common.gendir(dirn, 'cadir')
for fn in os.listdir(cadir):
if fn.endswith('.crt'):
shutil.copyfile(os.path.join(cadir, fn),
os.path.join(tlscadir, fn))
async with self.getTestCore(dirn=dirn) as core:
root = await core.auth.getUserByName('root')
await root.setPasswd('root')
addr, port = await core.addHttpsPort(0)
core.addHttpApi('/api/v0/test', s_test.HttpReflector,
{'cell': core})
url = f'https://*****:*****@localhost:{port}/api/v0/test'
opts = {'vars': {'url': url}}
q = '''
$params=((foo, bar), (key, valu))
$resp = $lib.inet.http.get($url, params=$params)
return ( ($resp.code, $resp.err) )
'''
code, (errname, _) = await core.callStorm(q, opts=opts)
self.eq(code, -1)
self.eq('ClientConnectorCertificateError', errname)
conf = {'tls:ca:dir': tlscadir}
async with self.getTestCore(dirn=dirn, conf=conf) as core:
addr, port = await core.addHttpsPort(0)
core.addHttpApi('/api/v0/test', s_test.HttpReflector,
{'cell': core})
url = f'https://*****:*****@localhost:{port}/api/v0/test'
opts = {'vars': {'url': url}}
q = '''
$params=((foo, bar), (key, valu))
$resp = $lib.inet.http.get($url, params=$params)
return ( $resp.json() )
'''
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {
'key': ('valu', ),
'foo': ('bar', )
})
def backup_lmdb(env, dstdir, txn=None):
tick = time.time()
s_common.gendir(dstdir)
env.copy(dstdir, compact=True, txn=txn)
tock = time.time()
logger.info(f'backup took: {tock-tick:.2f} seconds')
async def test_axon_tlscapath(self):
with self.getTestDir() as dirn:
cdir = s_common.gendir(dirn, 'certs')
cadir = s_common.gendir(cdir, 'cas')
tdir = s_certdir.CertDir(cdir)
tdir.genCaCert('somelocalca')
tdir.genHostCert('localhost', signas='somelocalca')
localkeyfp = tdir.getHostKeyPath('localhost')
localcertfp = tdir.getHostCertPath('localhost')
shutil.copyfile(localkeyfp, s_common.genpath(dirn, 'sslkey.pem'))
shutil.copyfile(localcertfp, s_common.genpath(dirn, 'sslcert.pem'))
tlscadir = s_common.gendir(dirn, 'cadir')
for fn in os.listdir(cadir):
if fn.endswith('.crt'):
shutil.copyfile(os.path.join(cadir, fn), os.path.join(tlscadir, fn))
conf = {'auth:passwd': 'root'}
async with self.getTestAxon(dirn=dirn, conf=conf) as axon:
host, port = await axon.addHttpsPort(0, host='127.0.0.1')
url = f'https://*****:*****@127.0.0.1:{port}/api/v1/active'
resp = await axon.wget(url)
self.false(resp.get('ok'))
self.isin('unable to get local issuer certificate', resp.get('mesg'))
retn = await axon.put(abuf)
self.eq(retn, asdfretn)
axon.addHttpApi('/api/v1/pushfile', HttpPushFile, {'cell': axon})
url = f'https://*****:*****@127.0.0.1:{port}/api/v1/pushfile'
resp = await axon.wput(asdfhash, url)
self.false(resp.get('ok'))
self.isin('unable to get local issuer certificate', resp.get('mesg'))
resp = await axon.postfiles(fields, url)
self.false(resp.get('ok'))
self.isin('unable to get local issuer certificate', resp.get('err'))
conf = {'auth:passwd': 'root', 'tls:ca:dir': tlscadir}
async with self.getTestAxon(dirn=dirn, conf=conf) as axon:
host, port = await axon.addHttpsPort(0, host='127.0.0.1')
url = f'https://*****:*****@localhost:{port}/api/v1/active'
resp = await axon.wget(url)
self.true(resp.get('ok'))
retn = await axon.put(abuf)
self.eq(retn, asdfretn)
axon.addHttpApi('/api/v1/pushfile', HttpPushFile, {'cell': axon})
url = f'https://*****:*****@localhost:{port}/api/v1/pushfile'
resp = await axon.wput(asdfhash, url)
self.true(resp.get('ok'))
resp = await axon.postfiles(fields, url)
self.true(resp.get('ok'))
async def _initCellSlab(self, readonly=False):
s_common.gendir(self.dirn, 'slabs')
path = os.path.join(self.dirn, 'slabs', 'cell.lmdb')
if not os.path.exists(path) and readonly:
logger.warning('Creating a slab for a readonly cell.')
_slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE)
_slab.initdb('hive')
await _slab.fini()
self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
self.onfini(self.slab.fini)
async def _initCellSlab(self, readonly=False):
s_common.gendir(self.dirn, 'slabs')
path = os.path.join(self.dirn, 'slabs', 'cell.lmdb')
if not os.path.exists(path) and readonly:
logger.warning('Creating a slab for a readonly cell.')
_slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE)
_slab.initdb('hive')
await _slab.fini()
self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
self.onfini(self.slab.fini)
async def test_layer_upstream_with_mirror(self):
with self.getTestDir() as dirn:
path00 = s_common.gendir(dirn, 'core00') # layer upstream
path01 = s_common.gendir(
dirn, 'core01') # layer downstream, mirror leader
path02 = s_common.gendir(
dirn, 'core02') # layer downstream, mirror follower
async with self.getTestCore(dirn=path00) as core00:
layriden = core00.view.layers[0].iden
await core00.nodes('[test:str=foobar +#hehe.haha]')
await core00.nodes('[ inet:ipv4=1.2.3.4 ]')
await core00.addTagProp('score', ('int', {}), {})
async with self.getTestCore(dirn=path01) as core01:
url = core00.getLocalUrl('*/layer')
conf = {'upstream': url}
ldef = await core01.addLayer(ldef=conf)
layr = core01.getLayer(ldef.get('iden'))
await core01.view.addLayer(layr.iden)
s_tools_backup.backup(path01, path02)
async with self.getTestCore(dirn=path01) as core01:
layr = core01.getLayer(ldef.get('iden'))
# Sync core01 with core00
offs = await core00.getView().layers[0].getNodeEditOffset()
evnt = await layr.waitUpstreamOffs(layriden, offs)
await asyncio.wait_for(evnt.wait(), timeout=8.0)
self.len(1, await core01.nodes('inet:ipv4=1.2.3.4'))
url = core01.getLocalUrl()
async with self.getTestCore(dirn=path02,
conf={'mirror':
url}) as core02:
await core02.sync()
layr = core01.getLayer(ldef.get('iden'))
self.true(layr.allow_upstream)
layr = core02.getLayer(ldef.get('iden'))
self.false(layr.allow_upstream)
self.len(1, await core02.nodes('inet:ipv4=1.2.3.4'))
async def __anit__(
self, # type: ignore
dirn: str,
opts: Optional[Dict] = None,
slabopts: Optional[Dict] = None):
'''
Args:
dirn (str): directory where to store the slabs
opts (Optional[Dict]): options for this multislab
slabopts (Optional[Dict]): options to pass through to the slab creation
'''
await s_base.Base.__anit__(self)
if opts is None:
opts = {}
self.offsevents: List[Tuple[int, int, asyncio.Event]] = [] # as a heap
self._waitcounter = 0
self.dirn: str = dirn
s_common.gendir(self.dirn)
self.slabopts: Dict[str, Any] = {} if slabopts is None else slabopts
# The last/current slab
self.tailslab: Optional[s_lmdbslab.Slab] = None
self.tailseqn: Optional[s_slabseqn.SlabSeqn] = None
# The most recently accessed slab/seqn that isn't the tail
self._cacheslab: Optional[s_lmdbslab.Slab] = None
self._cacheseqn: Optional[s_slabseqn.SlabSeqn] = None
self._cacheridx: Optional[int] = None
# A startidx -> (Slab, Seqn) dict for all open Slabs, so we don't accidentally open the same Slab twice
self._openslabs: Dict[int, Tuple[s_lmdbslab.Slab,
s_slabseqn.SlabSeqn]] = {}
# Lock to avoid an open race
self._openlock = asyncio.Lock()
await self._discoverRanges()
async def fini():
for slab, _ in list(self._openslabs.values()):
# We incref the slabs, so might have to fini multiple times
count = 1
while count:
count = await slab.fini()
self.onfini(fini)
def backup(srcdir, dstdir, compact=True):
'''
Args:
compact (bool): whether to optimize storage while copying to the destination
'''
tick = s_common.now()
srcdir = s_common.reqdir(srcdir)
dstdir = s_common.gendir(dstdir)
logger.info(f'Starting backup of [{srcdir}]')
logger.info(f'Destination dir: [{dstdir}]')
for root, dnames, fnames in os.walk(srcdir, topdown=True):
relpath = os.path.relpath(root, start=srcdir)
for name in list(dnames):
# Explicitly skip directory names of 'tmp' to avoid backing up temporary files
if name == 'tmp':
dnames.remove(name)
continue
srcpath = s_common.genpath(root, name)
dstpath = s_common.genpath(dstdir, relpath, name)
if name.endswith('.lmdb'):
dnames.remove(name)
backup_lmdb(srcpath, dstpath)
continue
logger.info(f'making dir:{dstpath}')
s_common.gendir(dstpath)
for name in fnames:
srcpath = s_common.genpath(root, name)
# skip unix sockets etc...
if not os.path.isfile(srcpath):
continue
dstpath = s_common.genpath(dstdir, relpath, name)
logger.info(f'copying: {srcpath} -> {dstpath}')
shutil.copy(srcpath, dstpath)
tock = s_common.now()
logger.info(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
return
async def test_lib_aha_offon(self):
with self.getTestDir() as dirn:
cryo0_dirn = s_common.gendir(dirn, 'cryo0')
conf = {'auth:passwd': 'secret'}
async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha:
host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
wait00 = aha.waiter(1, 'aha:svcadd')
cryo_conf = {
'aha:name': '0.cryo.mynet',
'aha:admin': '*****@*****.**',
'aha:registry': f'tcp://*****:*****@127.0.0.1:{port}',
'dmon:listen': 'tcp://0.0.0.0:0/',
}
async with self.getTestCryo(dirn=cryo0_dirn, conf=cryo_conf.copy()) as cryo:
self.len(1, await wait00.wait(timeout=6))
svc = await aha.getAhaSvc('0.cryo.mynet')
linkiden = svc.get('svcinfo', {}).get('online')
self.nn(linkiden)
# Tear down the Aha cell.
await aha.__aexit__(None, None, None)
async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha:
wait01 = aha.waiter(1, 'aha:svcdown')
await wait01.wait(timeout=6)
svc = await aha.getAhaSvc('0.cryo.mynet')
self.notin('online', svc.get('svcinfo'))
# Try setting something down a second time
await aha.setAhaSvcDown('0.cryo.mynet', linkiden, network=None)
svc = await aha.getAhaSvc('0.cryo.mynet')
self.notin('online', svc.get('svcinfo'))
async def _initCellAuth(self):
if not self.boot.get('auth:en'):
return
# runtime import. dep loop.
import synapse.cells as s_cells
authdir = s_common.gendir(self.dirn, 'auth')
self.auth = await s_cells.init('auth', authdir)
admin = self.boot.get('auth:admin')
if admin is not None:
name, passwd = admin.split(':')
user = self.auth.users.get(name)
if user is None:
user = self.auth.addUser(name)
logger.warning(f'adding admin user: {name} to {self.cellname}')
user.setAdmin(True)
user.setPasswd(passwd)
self.onfini(self.auth.fini)
async def __anit__(self, dirn: str, conf=None) -> None:
await s_cell.Cell.__anit__(self, dirn)
if conf is not None:
self.conf.update(conf)
path = s_common.gendir(self.dirn, 'tank.lmdb')
mapsize = self.conf.get('mapsize')
self.lenv = lmdb.open(path, writemap=True, max_dbs=128)
self.lenv.set_mapsize(mapsize)
self.lenv_items = self.lenv.open_db(b'items')
self.lenv_metrics = self.lenv.open_db(b'metrics')
offsdb = self.lenv.open_db(b'offsets')
self.offs = s_lmdb.Offs(self.lenv, offsdb)
noindex = self.conf.get('noindex')
self.indexer = None if noindex else CryoTankIndexer(self)
with self.lenv.begin() as xact:
self.items_indx = xact.stat(self.lenv_items)['entries']
self.metrics_indx = xact.stat(self.lenv_metrics)['entries']
def fini():
self.lenv.close()
self.onfini(fini)
async def __anit__(self, dirn):
await s_cell.Cell.__anit__(self, dirn)
path = s_common.gendir(self.dirn, 'cryo.lmdb')
self.dmon = None
self.sharename = None
self.kvstor = s_kv.KvStor(path)
self.onfini(self.kvstor.fini)
self.names = self.kvstor.getKvDict('cryo:names')
self.confs = self.kvstor.getKvDict('cryo:confs')
self.tanks = await s_base.BaseRef.anit()
self.onfini(self.tanks.fini)
for name, iden in self.names.items():
logger.info('Bringing tank [%s][%s] online', name, iden)
path = s_common.genpath(self.dirn, 'tanks', iden)
conf = self.confs.get(name)
tank = await CryoTank.anit(path, conf)
self.tanks.put(name, tank)
def _fireAxonIden(self, iden):
axondir = s_common.gendir(self.datadir, '%s.axon' % iden)
opts = self.makeAxonOpts()
jsopts = s_common.jsload(axondir, 'axon.opts')
if jsopts is not None:
opts.update(jsopts)
# Special case where the axonbus may update - we want to ensure
# we're passing the latest axonbus to the Axon so it can register
# itself properly.
axonbus = opts.get('axon:axonbus')
if axonbus is not None:
myaxonbus = self.getConfOpt('axon:axonbus')
if axonbus != myaxonbus:
opts['axon:axonbus'] = myaxonbus
s_common.jssave(opts, axondir, 'axon.opts')
self.axons[iden] = Axon(axondir, **opts)
bytemax = opts.get('axon:bytemax')
clone = opts.get('axon:clone')
if clone:
self.cloneaxons.append(iden)
self.usedspace = self.usedspace + bytemax
def __init__(self, path, **opts):
s_eventbus.EventBus.__init__(self)
self.size = 0
self.opts = opts
self.last = None
self.path = s_common.gendir(path)
self.lock = threading.Lock()
self.window = collections.deque()
self.opts.setdefault('filemax', gigabyte)
self.baseoff = opts.get('base', 0) # base address of this file
self.files = []
self.pumps = {}
self.queues = []
self.pumpers = []
self._initPersFiles()
if self.files:
self.last = self.files[-1]
if self.last is None:
self.last = self._addPersFile(0)
self.onfini(self._onDirFini)
async def getTestReadWriteCores(self, conf=None, dirn=None):
dirn0 = None
if dirn is not None:
dirn0 = s_common.gendir(dirn, 'remotecore')
async with self.getRemoteCores(dirn0=dirn0, conf1=conf, dirn1=dirn) as (core0, core1):
yield core1, core0
async def __anit__(self, dirn, conf=None): # type: ignore
await s_cell.Cell.__anit__(self, dirn, conf=conf)
# share ourself via the cell dmon as "axon"
# for potential default remote use
self.dmon.share('axon', self)
path = s_common.gendir(self.dirn, 'axon.lmdb')
self.axonslab = await s_lmdbslab.Slab.anit(path)
self.sizes = self.axonslab.initdb('sizes')
self.onfini(self.axonslab.fini)
self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history')
self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn')
node = await self.hive.open(('axon', 'metrics'))
self.axonmetrics = await node.dict()
self.axonmetrics.setdefault('size:bytes', 0)
self.axonmetrics.setdefault('file:count', 0)
self.addHealthFunc(self._axonHealth)
# modularize blob storage
await self._initBlobStor()
self._initAxonHttpApi()
def initCellAuth(self):
# To avoid import cycle
import synapse.lib.auth as s_auth
valu = self.boot.get('auth:en')
if not valu:
return
url = self.boot.get('auth:url')
if url is not None:
self.auth = s_telepath.openurl(url)
return
# setup local auth
dirn = s_common.gendir(self.dirn, 'auth')
self.auth = s_auth.Auth(dirn) # FIXME this is not imported, but would cause circular import
# let them hard code an initial admin user:passwd
admin = self.boot.get('auth:admin')
if admin is not None:
name, passwd = admin.split(':', 1)
user = self.auth.getUser(name)
if user is None:
user = self.auth.addUser(name)
user.setAdmin(True)
user.setPasswd(passwd)
async def test_csvtool_missingvals(self):
async with self.getTestCore() as core:
url = core.getLocalUrl()
dirn = s_common.gendir(core.dirn, 'junk')
logpath = s_common.genpath(dirn, 'csvtest.log')
csvpath = s_common.genpath(dirn, 'csvtest.csv')
with s_common.genfile(csvpath) as fd:
fd.write(csvfile_missing)
stormpath = s_common.genpath(dirn, 'csvtest.storm')
with s_common.genfile(stormpath) as fd:
fd.write(csvstorm_missing)
argv = [
'--csv-header', '--debug', '--cortex', url, '--logfile',
logpath, stormpath, csvpath
]
outp = self.getTestOutp()
await s_csvtool.main(argv, outp=outp)
outp.expect('hello hello')
outp.expect("'fqdn': 'google.com'")
outp.expect('3 nodes')
async def test_basics(self):
async with self.getTestCore() as core: # type: s_cortex.Cortex
testmod = core.getCoreMod('synapse.tests.utils.TestModule')
self.isinstance(testmod, s_module.CoreModule)
# modname from class name
self.eq(testmod.mod_name, 'testmodule')
foomod = await core.loadCoreModule(foo_ctor)
# modname from explicit modname
self.eq(foomod.mod_name, 'foo')
# modpaths are dynamically made on demand
self.false(os.path.isdir(foomod._modpath))
mpath = foomod.getModPath()
self.isin(os.path.join('mods', 'foo'), mpath)
self.true(os.path.isdir(foomod._modpath))
# preload a config file for the BarModule
dirn = s_common.gendir(core.dirn, 'mods', 'barmod')
s_common.yamlsave({'test': 1, 'duck': 'quack'}, dirn, 'conf.yaml')
# barmodule loads a layerctor
self.false(core.layrctors.get('newp') is int)
barmod = await core.loadCoreModule(bar_ctor)
self.eq(barmod.data, {})
self.eq(barmod.conf, {
'test': 1,
'hehe': 'haha',
'duck': 'quack',
})
self.true(core.layrctors.get('newp') is int)
self.eq(barmod.data, {'fini': True})
async def test_telepath_loadenv(self):
with self.getTestDir() as dirn:
certpath = s_common.gendir(dirn, 'certs')
newppath = s_common.genpath(dirn, 'newps')
conf = {
'version': 1,
'aha:servers': [
'tcp://localhost:9999/',
],
'certdirs': [
certpath,
newppath,
],
}
path = s_common.genpath(dirn, 'telepath.yaml')
s_common.yamlsave(conf, path)
fini = await s_telepath.loadTeleEnv(path)
await fini()
self.none(await s_telepath.loadTeleEnv(newppath))
conf['version'] = 99
s_common.yamlsave(conf, path)
self.none(await s_telepath.loadTeleEnv(path))
async def getTestReadWriteCores(self, conf=None, dirn=None):
dirn0 = None
if dirn is not None:
dirn0 = s_common.gendir(dirn, 'remotecore')
async with self.getRemoteCores(dirn0=dirn0, conf1=conf, dirn1=dirn) as (core0, core1):
yield core1, core0
async def test_csvtool(self):
async with self.getTestCore() as core:
url = core.getLocalUrl()
dirn = s_common.gendir(core.dirn, 'junk')
logpath = s_common.genpath(dirn, 'csvtest.log')
csvpath = s_common.genpath(dirn, 'csvtest.csv')
with s_common.genfile(csvpath) as fd:
fd.write(csvfile)
stormpath = s_common.genpath(dirn, 'csvtest.storm')
with s_common.genfile(stormpath) as fd:
fd.write(csvstorm)
argv = [
'--csv-header', '--debug', '--cortex', url, '--logfile',
logpath, stormpath, csvpath
]
outp = self.getTestOutp()
await s_csvtool.main(argv, outp=outp)
outp.expect('oh hai')
outp.expect('2 nodes')
async def test_csvtool(self):
async with self.getTestCore() as core:
url = core.getLocalUrl()
dirn = s_common.gendir(core.dirn, 'junk')
logpath = s_common.genpath(dirn, 'csvtest.log')
csvpath = s_common.genpath(dirn, 'csvtest.csv')
with s_common.genfile(csvpath) as fd:
fd.write(csvfile)
stormpath = s_common.genpath(dirn, 'csvtest.storm')
with s_common.genfile(stormpath) as fd:
fd.write(csvstorm)
argv = [
'--csv-header', '--debug', '--cortex', url, '--logfile',
logpath, stormpath, csvpath
]
outp = self.getTestOutp()
await s_csvtool.main(argv, outp=outp)
outp.expect('oh hai')
outp.expect('2 nodes')
with mock.patch('synapse.telepath.Proxy._getSynVers',
self._getOldSynVers):
outp = self.getTestOutp()
await s_csvtool.main(argv, outp=outp)
outp.expect(
'Cortex version 0.0.0 is outside of the csvtool supported range'
)
def __init__(self, dirn, mapsize=s_const.tebibyte): # from LMDB docs....
s_eventbus.EventBus.__init__(self)
path = s_common.gendir(dirn, 'axon.lmdb')
self.lmdb = lmdb.open(path, writemap=True, max_dbs=128)
self.lmdb.set_mapsize(mapsize)
self.lmdb_bytes = self.lmdb.open_db(b'bytes')
self.lmdb_files = self.lmdb.open_db(b'files')
self.lmdb_fileblocks = self.lmdb.open_db(b'fileblocks')
self.lmdb_hashes = self.lmdb.open_db(b'hashes', dupsort=True)
with self.lmdb.begin() as xact:
self.bytes_indx = xact.stat(db=self.lmdb_bytes)['entries']
self.files_indx = xact.stat(db=self.lmdb_files)['entries']
def fini():
self.lmdb.sync()
self.lmdb.close()
self.onfini(fini)
self.inprog = {}
async def getTestCore(self, conf=None, dirn=None):
# make remote core from provided dirn for repeatability
dirn0 = None
if dirn is not None:
dirn0 = s_common.gendir(dirn, 'remotecore')
async with self.getRemoteCores(dirn0=dirn0, conf1=conf, dirn1=dirn) as (core0, core1):
yield core1
def getModDir(self):
'''
Get the path to the module specific directory.
Notes:
This creates the directory if it did not previously exist.
Returns:
str: The filepath to the module specific directory.
'''
return s_common.gendir(self._modpath)
def backup(srcdir, dstdir):
tick = s_common.now()
srcdir = s_common.reqdir(srcdir)
dstdir = s_common.gendir(dstdir)
logger.info(f'Starting backup of [{srcdir}]')
logger.info(f'Destination dir: [{dstdir}]')
for root, dnames, fnames in os.walk(srcdir, topdown=True):
relpath = os.path.relpath(root, start=srcdir)
for name in list(dnames):
srcpath = s_common.genpath(root, name)
dstpath = s_common.genpath(dstdir, relpath, name)
if name.endswith('.lmdb'):
dnames.remove(name)
backup_lmdb(srcpath, dstpath)
continue
logger.info(f'making dir:{dstpath}')
s_common.gendir(dstpath)
for name in fnames:
srcpath = s_common.genpath(root, name)
# skip unix sockets etc...
if not os.path.isfile(srcpath):
continue
dstpath = s_common.genpath(dstdir, relpath, name)
logger.info(f'copying: {srcpath} -> {dstpath}')
shutil.copy(srcpath, dstpath)
tock = s_common.now()
logger.info(f'Backup complete. Took [{tock-tick:.2f}] for [{srcdir}]')
return
def main(argv, outp=None):
pars = setup()
opts = pars.parse_args(argv)
if outp is None: # pragma: no cover
outp = s_output.OutPut()
if opts.output is None:
opts.output = '.'
outdir = pathlib.Path(opts.output)
s_common.gendir(opts.output)
with s_telepath.openurl(opts.axon) as axon:
# reminder: these are the hashes *not* available
awants = axon.wants([binascii.unhexlify(h) for h in opts.hashes])
for a in awants:
outp.printf(f'{binascii.hexlify(a)} not in axon store')
exists = [h for h in opts.hashes if binascii.unhexlify(h) not in awants]
for h in exists:
try:
outp.printf(f'Fetching {h} to file')
with open(outdir.joinpath(h), 'wb') as fd:
for b in axon.get(binascii.unhexlify(h)):
fd.write(b)
outp.printf(f'Fetched {h} to file')
except Exception as e:
outp.printf('Error: Hit Exception: %s' % (str(e),))
continue
return 0
async def __anit__(self, dirn, conf=None):
await s_base.Base.__anit__(self)
if conf is None:
conf = {}
self.conf = conf
self.dirn = s_common.gendir(dirn)
self._iden = self._getTankIden()
path = s_common.gendir(self.dirn, 'tank.lmdb')
self.slab = await s_lmdbslab.Slab.anit(path)
self.offs = s_lmdbslab.Offs(self.slab, 'offsets')
self._items = s_slabseqn.SlabSeqn(self.slab, 'items')
self._metrics = s_slabseqn.SlabSeqn(self.slab, 'metrics')
self.onfini(self.slab.fini)
async def __anit__(self, dirn, conf=None):
await s_cell.Cell.__anit__(self, dirn)
path = s_common.gendir(self.dirn, 'axon.lmdb')
self.axonslab = await s_lmdbslab.Slab.anit(path)
self.sizes = self.axonslab.initdb('sizes')
self.onfini(self.axonslab.fini)
self.axonhist = s_lmdbslab.Hist(self.axonslab, 'history')
path = s_common.gendir(self.dirn, 'blob.lmdb')
self.blobslab = await s_lmdbslab.Slab.anit(path)
self.blobs = self.blobslab.initdb('blobs')
self.onfini(self.blobslab.fini)
self.axonseqn = s_slabseqn.SlabSeqn(self.axonslab, 'axonseqn')
node = await self.hive.open(('axon', 'metrics'))
self.axonmetrics = await node.dict()
self.axonmetrics.setdefault('size:bytes', 0)
self.axonmetrics.setdefault('file:count', 0)
async def __anit__(self, core, node):
await s_base.Base.__anit__(self)
self.core = core
self.node = node
self.iden = node.name()
self.buidcache = s_cache.LruDict(BUID_CACHE_SIZE)
# splice windows...
self.windows = []
self.info = await node.dict()
self.info.setdefault('owner', 'root')
self.owner = self.info.get('owner')
self.conf = await (await node.open(('config',))).dict()
for name, info in self.confdefs:
dval = info.get('defval', s_common.novalu)
if dval is s_common.novalu:
continue
self.conf.setdefault(name, dval)
self.dirn = s_common.gendir(core.dirn, 'layers', self.iden)
self._lift_funcs = {
'indx': self._liftByIndx,
'prop:re': self._liftByPropRe,
'univ:re': self._liftByUnivRe,
'form:re': self._liftByFormRe,
'prop:ival': self._liftByPropIval,
'univ:ival': self._liftByUnivIval,
'form:ival': self._liftByFormIval,
}
self._stor_funcs = {
'prop:set': self._storPropSet,
'prop:del': self._storPropDel,
'buid:set': self._storBuidSet,
}
self.fresh = False
self.canrev = True
self.spliced = asyncio.Event(loop=self.loop)
self.onfini(self.spliced.set)
self.onfini(self._onLayrFini)
def __init__(self, path=None):
self.crypto_numbits = 4096
self.signing_digest = 'sha256'
if path is None:
path = defdir
s_common.gendir(path, 'cas')
s_common.gendir(path, 'hosts')
s_common.gendir(path, 'users')
self.certdir = s_common.reqdir(path)
self.path = path
async def __anit__(self, dirn):
await s_cell.Cell.__anit__(self, dirn)
self.dmon.share('cryotank', self)
path = s_common.gendir(self.dirn, 'cryo.lmdb')
self.names = await self.hive.open(('cryo', 'names'))
self.tanks = await s_base.BaseRef.anit()
self.onfini(self.tanks.fini)
for name, node in self.names:
iden, conf = node.valu
logger.info('Bringing tank [%s][%s] online', name, iden)
path = s_common.genpath(self.dirn, 'tanks', iden)
tank = await CryoTank.anit(path, conf)
self.tanks.put(name, tank)
async def test_csvtool(self):
async with self.getTestCore() as core:
url = core.getLocalUrl()
dirn = s_common.gendir(core.dirn, 'junk')
logpath = s_common.genpath(dirn, 'csvtest.log')
csvpath = s_common.genpath(dirn, 'csvtest.csv')
with s_common.genfile(csvpath) as fd:
fd.write(csvfile)
stormpath = s_common.genpath(dirn, 'csvtest.storm')
with s_common.genfile(stormpath) as fd:
fd.write(csvstorm)
argv = ['--csv-header', '--debug', '--cortex', url, '--logfile', logpath, stormpath, csvpath]
outp = self.getTestOutp()
await s_csvtool.main(argv, outp=outp)
outp.expect('2 nodes (9 created)')
async def __anit__(self, dirn, conf=None, readonly=False):
await s_base.Base.__anit__(self)
s_telepath.Aware.__init__(self)
self.dirn = s_common.gendir(dirn)
self.auth = None
# each cell has a guid
path = s_common.genpath(dirn, 'cell.guid')
# generate a guid file if needed
if not os.path.isfile(path):
with open(path, 'w') as fd:
fd.write(s_common.guid())
# read our guid file
with open(path, 'r') as fd:
self.iden = fd.read().strip()
boot = self._loadCellYaml('boot.yaml')
self.boot = s_common.config(boot, bootdefs)
await self._initCellDmon()
if conf is None:
conf = {}
[conf.setdefault(k, v) for (k, v) in self._loadCellYaml('cell.yaml').items()]
self.conf = s_common.config(conf, self.confdefs + self.confbase)
self.cmds = {}
self.insecure = self.boot.get('insecure', False)
self.sessions = {}
self.httpsonly = self.conf.get('https:only', False)
self.boss = await s_boss.Boss.anit()
self.onfini(self.boss)
await self._initCellSlab(readonly=readonly)
self.hive = await self._initCellHive()
self.auth = await self._initCellAuth()
# check and migrate old cell auth
oldauth = s_common.genpath(self.dirn, 'auth')
if os.path.isdir(oldauth):
await s_compat.cellAuthToHive(oldauth, self.auth)
os.rename(oldauth, oldauth + '.old')
admin = self.boot.get('auth:admin')
if admin is not None:
name, passwd = admin.split(':', 1)
user = self.auth.getUserByName(name)
if user is None:
user = await self.auth.addUser(name)
await user.setAdmin(True)
await user.setPasswd(passwd)
self.insecure = False
await self._initCellHttp()
async def fini():
[await s.fini() for s in self.sessions.values()]
self.onfini(fini)
def main(argv):
pars = argparse.ArgumentParser(prog='easycert', description=descr)
pars.add_argument('--certdir', default='~/.syn/certs', help='Directory for certs/keys')
pars.add_argument('--signas', help='sign the new cert with the given cert name')
pars.add_argument('--ca', default=False, action='store_true', help='mark the certificate as a CA/CRL signer')
pars.add_argument('--server', default=False, action='store_true', help='mark the certificate as a server')
pars.add_argument('name', help='common name for the certificate')
opts = pars.parse_args(argv)
certdir = gendir(opts.certdir)
pkeypath = os.path.join(certdir,'%s.key' % opts.name)
certpath = os.path.join(certdir,'%s.crt' % opts.name)
if os.path.exists(pkeypath):
print('key exists: %s' % (pkeypath,))
return(-1)
if os.path.exists(certpath):
print('cert exists: %s' % (certpath,))
return(-1)
pkey = crypto.PKey()
pkey.generate_key(crypto.TYPE_RSA, 2048)
cert = crypto.X509()
cert.set_pubkey(pkey)
cert.gmtime_adj_notBefore(0)
cert.gmtime_adj_notAfter(10*365*24*60*60)
cert.set_serial_number( int(time.time()) )
if opts.ca:
ext0 = crypto.X509Extension(b'basicConstraints',False,b'CA:TRUE')
cert.add_extensions([ext0])
else:
keyuse = [b'digitalSignature']
extuse = [b'clientAuth']
certtype = b'client'
if opts.server:
certtype = b'server'
extuse = [b'serverAuth']
keyuse.append(b'keyEncipherment')
ext0 = crypto.X509Extension(b'nsCertType',False,certtype)
ext1 = crypto.X509Extension(b'keyUsage',False,b','.join(keyuse))
extuse = b','.join(extuse)
ext2 = crypto.X509Extension(b'extendedKeyUsage',False,extuse)
ext3 = crypto.X509Extension(b'basicConstraints',False,b'CA:FALSE')
cert.add_extensions([ext0,ext1,ext2,ext3])
subj = cert.get_subject()
subj.CN = opts.name
signcert = cert
signpkey = pkey
if opts.signas:
path = os.path.join(certdir,'%s.key' % (opts.signas,))
byts = open(path,'rb').read()
signpkey = crypto.load_privatekey(crypto.FILETYPE_PEM, byts)
path = os.path.join(certdir,'%s.crt' % (opts.signas,))
byts = open(path,'rb').read()
signcert = crypto.load_certificate(crypto.FILETYPE_PEM, byts)
cert.set_issuer( signcert.get_subject() )
cert.sign( signpkey, 'sha1' )
byts = crypto.dump_privatekey(crypto.FILETYPE_PEM, pkey)
with genfile(pkeypath) as fd:
fd.write(byts)
print('pkey saved: %s' % (pkeypath,))
byts = crypto.dump_certificate(crypto.FILETYPE_PEM, cert)
with genfile(certpath) as fd:
fd.write(byts)
print('cert saved: %s' % (certpath,))
