async def test_syningest_remote(self):
async with self.getTestCore() as core:
guid = s_common.guid()
seen = s_common.now()
gestdef = self.getIngestDef(guid, seen)
with self.getTestDir() as dirn:
# Test yaml support here
gestfp = s_common.genpath(dirn, 'gest.yaml')
s_common.yamlsave(gestdef, gestfp)
argv = ['--cortex', core.getLocalUrl(),
'--debug',
'--modules', 'synapse.tests.utils.TestModule',
gestfp]
outp = self.getTestOutp()
cmdg = s_t_utils.CmdGenerator(['storm test:pivcomp -> *', EOFError()])
with self.withCliPromptMockExtendOutp(outp):
with self.withTestCmdr(cmdg):
self.eq(await s_feed.main(argv, outp=outp), 0)
self.true(outp.expect('test:str=haha', throw=False))
self.true(outp.expect('test:pivtarg=hehe', throw=False))
async def test_cell_nonstandard_admin(self):
boot = {
'auth:admin': 'pennywise:cottoncandy',
'type': 'echoauth',
}
pconf = {'user': '******', 'passwd': 'cottoncandy'}
with self.getTestDir('cellauth') as dirn:
s_common.yamlsave(boot, dirn, 'boot.yaml')
async with await EchoAuth.anit(dirn) as echo:
echo.insecure = False
# start a regular network listener so we can auth
host, port = await echo.dmon.listen('tcp://127.0.0.1:0/')
async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/', **pconf) as proxy:
self.true(await proxy.isadmin())
self.true(await proxy.allowed(('hehe', 'haha')))
url = f'tcp://[email protected]:{port}/'
await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url))
# Ensure the cell and its auth have been fini'd
self.true(echo.isfini)
self.true(echo.auth.isfini)
self.true(echo.auth.getUserByName('root').isfini)
self.true(echo.auth.getUserByName('pennywise').isfini)
def testmain():
pconf = {'user': '******', 'passwd': 'root'}
with self.getTestProxy(dmon, 'core', **pconf) as core:
# Setup user permissions
core.addAuthRole('creator')
core.addAuthRule('creator', (True, ('node:add', )))
core.addAuthRule('creator', (True, ('prop:set', )))
core.addAuthRule('creator', (True, ('tag:add', )))
core.addUserRole('root', 'creator')
host, port = dmon.addr
curl = f'tcp://*****:*****@{host}:{port}/core'
guid = s_common.guid()
seen = s_common.now()
gestdef = self.getIngestDef(guid, seen)
with self.getTestDir() as dirn:
# Test yaml support here
gestfp = s_common.genpath(dirn, 'gest.yaml')
s_common.yamlsave(gestdef, gestfp)
argv = [
'--cortex', curl, '--debug', '--modules',
'synapse.tests.utils.TestModule', gestfp
]
outp = self.getTestOutp()
cmdg = s_t_utils.CmdGenerator(['storm pivcomp -> *'],
on_end=EOFError)
with mock.patch('synapse.lib.cli.get_input', cmdg):
self.eq(s_feed.main(argv, outp=outp), 0)
self.true(outp.expect('teststr=haha', throw=False))
self.true(outp.expect('pivtarg=hehe', throw=False))
async def test_load_failures(self):
async with self.getTestCore() as core: # type: s_cortex.Cortex
with self.setTstEnvars(SYN_TEST_MOD_FAIL_PRE=1) as cm:
with self.getAsyncLoggerStream('synapse.cortex',
'preCoreModuleFail') as stream:
self.none(await core.loadCoreModule(foo_ctor))
self.true(await stream.wait(1))
self.none(core.getCoreMod(foo_ctor))
with self.setTstEnvars(SYN_TEST_MOD_FAIL_INIT=1) as cm:
with self.getAsyncLoggerStream('synapse.cortex',
'initCoreModuleFail') as stream:
self.none(await core.loadCoreModule(foo_ctor))
self.true(await stream.wait(1))
self.none(core.getCoreMod(foo_ctor))
with self.getTestDir(mirror='testcore') as dirn:
conf = s_common.yamlload(dirn, 'cell.yaml')
conf['modules'].append(foo_ctor)
s_common.yamlsave(conf, dirn, 'cell.yaml')
conf = s_common.yamlload(dirn, 'cell.yaml')
with self.setTstEnvars(SYN_TEST_MOD_FAIL_PRE=1) as cm:
with self.getAsyncLoggerStream('synapse.cortex',
'preCoreModuleFail') as stream:
async with await s_cortex.Cortex.anit(dirn) as core:
self.true(await stream.wait(1))
self.none(core.getCoreMod(foo_ctor))
with self.setTstEnvars(SYN_TEST_MOD_FAIL_INIT=1) as cm:
with self.getAsyncLoggerStream('synapse.cortex',
'initCoreModuleFail') as stream:
async with await s_cortex.Cortex.anit(dirn) as core:
self.true(await stream.wait(1))
self.none(core.getCoreMod(foo_ctor))
def test_common_yaml(self):
obj = [{'key': 1,
'key2': [1, 2, 3],
'key3': True,
'key4': 'some str',
'key5': {
'oh': 'my',
'we all': 'float down here'
}, },
'duck',
False,
'zero',
0.1,
]
with self.getTestDir() as dirn:
s_common.yamlsave(obj, dirn, 'test.yaml')
robj = s_common.yamlload(dirn, 'test.yaml')
self.eq(obj, robj)
obj = {'foo': 'bar', 'zap': [3, 4, 'f']}
s_common.yamlsave(obj, dirn, 'test.yaml')
s_common.yamlmod({'bar': 42}, dirn, 'test.yaml')
robj = s_common.yamlload(dirn, 'test.yaml')
obj['bar'] = 42
self.eq(obj, robj)
# Test yaml helper safety
s = '!!python/object/apply:os.system ["pwd"]'
with s_common.genfile(dirn, 'explode.yaml') as fd:
fd.write(s.encode())
self.raises(yaml.YAMLError, s_common.yamlload, dirn, 'explode.yaml')
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.save',
description='Save tree data from a remote hive to file.')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to save.')
opts = pars.parse_args(argv)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
tree = await hive.saveHiveTree(path=path)
if opts.yaml:
s_common.yamlsave(tree, opts.filepath)
else:
s_msgpack.dumpfile(tree, opts.filepath)
async def test_cell_hiveboot(self):
with self.getTestDir() as dirn:
tree = {
'kids': {
'hehe': {
'value': 'haha'
},
}
}
bootpath = os.path.join(dirn, 'hiveboot.yaml')
s_common.yamlsave(tree, bootpath)
async with await s_cell.Cell.anit(dirn) as cell:
self.eq('haha', await cell.hive.get(('hehe', )))
# test that the file does not load again
tree['kids']['redbaloons'] = {'value': 99}
s_common.yamlsave(tree, bootpath)
async with await s_cell.Cell.anit(dirn) as cell:
self.none(await cell.hive.get(('redbaloons', )))
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
async def test_cell_nonstandard_admin(self):
boot = {
'auth:admin': 'pennywise:cottoncandy',
'type': 'echoauth',
}
pconf = {'user': '******', 'passwd': 'cottoncandy'}
with self.getTestDir('cellauth') as dirn:
s_common.yamlsave(boot, dirn, 'boot.yaml')
async with await EchoAuth.anit(dirn) as echo:
echo.insecure = False
# start a regular network listener so we can auth
host, port = await echo.dmon.listen('tcp://127.0.0.1:0/')
async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/',
**pconf) as proxy:
self.true(await proxy.isadmin())
self.true(await proxy.allowed('hehe', 'haha'))
url = f'tcp://[email protected]:{port}/'
await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url))
# Ensure the cell and its auth have been fini'd
self.true(echo.isfini)
self.true(echo.auth.isfini)
self.true(echo.auth.getUserByName('root').isfini)
self.true(echo.auth.getUserByName('pennywise').isfini)
async def test_server_mirror_badiden(self):
with self.getTestDir() as dirn, self.withSetLoggingMock():
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
argv0 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--name', 'srccore',
path00,
]
out0 = self.getTestOutp()
async with await s_cortex.Cortex.initFromArgv(argv0, outp=out0) as core00:
out1 = self.getTestOutp()
argv1 = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--mirror', core00.getLocalUrl(),
path01,
]
s_common.yamlsave({'logchanges': True}, path01, 'cell.yaml')
async with await s_cortex.Cortex.initFromArgv(argv1, outp=out1) as core01:
await core01.waitfini(6)
self.true(core01.isfini)
async def test_basics(self):
async with self.getTestCore() as core: # type: s_cortex.Cortex
testmod = core.getCoreMod('synapse.tests.utils.TestModule')
self.isinstance(testmod, s_module.CoreModule)
# modname from class name
self.eq(testmod.mod_name, 'testmodule')
foomod = await core.loadCoreModule(foo_ctor)
# modname from explicit modname
self.eq(foomod.mod_name, 'foo')
# modpaths are dynamically made on demand
self.false(os.path.isdir(foomod._modpath))
mpath = foomod.getModPath()
self.isin(os.path.join('mods', 'foo'), mpath)
self.true(os.path.isdir(foomod._modpath))
# preload a config file for the BarModule
dirn = s_common.gendir(core.dirn, 'mods', 'barmod')
s_common.yamlsave({'test': 1, 'duck': 'quack'}, dirn, 'conf.yaml')
# barmodule loads a layerctor
self.false(core.layrctors.get('newp') is int)
barmod = await core.loadCoreModule(bar_ctor)
self.eq(barmod.data, {})
self.eq(barmod.conf, {
'test': 1,
'hehe': 'haha',
'duck': 'quack',
})
self.true(core.layrctors.get('newp') is int)
self.eq(barmod.data, {'fini': True})
def test_doc_data(self):
with self.getTestDir() as dirn:
s_common.gendir(dirn, 'docdata', 'stuff')
docdata = s_common.genpath(dirn, 'docdata')
root = s_common.genpath(dirn, 'synapse', 'userguides')
d = {'key': 'value'}
s_common.jssave(d, docdata, 'data.json')
s_common.yamlsave(d, docdata, 'data.yaml')
s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk'))
with s_common.genfile(docdata, 'stuff', 'data.txt') as fd:
fd.write('beep'.encode())
with s_common.genfile(docdata, 'data.jsonl') as fd:
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
fd.write(json.dumps(d).encode() + b'\n')
data = s_jupyter.getDocData('data.json', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.yaml', root)
self.eq(data, d)
data = s_jupyter.getDocData('data.mpk', root)
self.eq(data, d)
data = s_jupyter.getDocData('stuff/data.txt', root)
self.eq(data, b'beep')
data = s_jupyter.getDocData('data.jsonl', root)
self.eq(data, [d, d, d])
self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root)
self.raises(ValueError, s_jupyter.getDocData,
'../../../../../../etc/passwd', root)
async def test_cell_confprint(self):
with self.withSetLoggingMock():
with self.getTestDir() as dirn:
conf = {
'dmon:listen': 'tcp://127.0.0.1:0',
'https:port': 0,
}
s_common.yamlsave(conf, dirn, 'cell.yaml')
outp = self.getTestOutp()
async with await s_cell.Cell.initFromArgv([dirn],
outp=outp) as cell:
outp.expect('...cell API (telepath): tcp://127.0.0.1:0')
outp.expect('...cell API (https): 0')
conf = {
'dmon:listen': 'tcp://127.0.0.1:0',
'https:port': None,
}
s_common.yamlsave(conf, dirn, 'cell.yaml')
outp = self.getTestOutp()
async with await s_cell.Cell.initFromArgv([dirn],
outp=outp) as cell:
outp.expect('...cell API (telepath): tcp://127.0.0.1:0')
outp.expect('...cell API (https): disabled')
async def test_lib_aha_loadenv(self):
with self.getTestDir() as dirn:
async with self.getTestAha() as aha:
host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
await aha.auth.rootuser.setPasswd('hehehaha')
conf = {
'version': 1,
'aha:servers': [
f'tcp://*****:*****@127.0.0.1:{port}/',
],
}
path = s_common.genpath(dirn, 'telepath.yaml')
s_common.yamlsave(conf, path)
fini = await s_telepath.loadTeleEnv(path)
# Should be one uninitialized aha client
self.len(1, s_telepath.aha_clients)
[info] = s_telepath.aha_clients.values()
self.none(info.get('client'))
with self.raises(s_exc.NoSuchName):
await s_telepath.openurl('aha://[email protected]')
# Connecting to an aha url should have initialized the client
self.len(1, s_telepath.aha_clients)
self.nn(info.get('client'))
await fini()
async def test_migr_yamlmod(self):
with self.getRegrDir('cortexes', REGR_VER) as src:
with self.getTestDir() as dest:
locallyrs = os.listdir(os.path.join(src, 'layers'))
mods = {'modules': ['synapse.tests.utils.TestModule']}
s_common.yamlsave(mods, src, 'cell.yaml')
conf = {
'src': src,
'dest': dest,
'migrops': ['dmodel', 'hivelyr', 'nodes'],
'editbatchsize': 1,
'nodelim': 2,
}
async with await s_migr.Migrator.anit(conf) as migr:
# test that we can skip dirn migration if it already exists
migr.migrops.append('dirn')
await migr._migrDirn()
del migr.migrops[-1]
# verify that test:int was loaded and migrated
await migr.migrate()
stats = []
for iden in locallyrs:
statkey = f'{iden}:form:test:int'
stats.extend([log async for log in migr._migrlogGet('nodes', 'stat', statkey)])
self.eq((1, 1), stats[0]['val'])
async def test_cell_confprint(self):
with self.withSetLoggingMock():
with self.getTestDir() as dirn:
conf = {
'dmon:listen': 'tcp://127.0.0.1:0',
'https:port': 0,
}
s_common.yamlsave(conf, dirn, 'cell.yaml')
with self.getAsyncLoggerStream('synapse.lib.cell') as stream:
async with await s_cell.Cell.initFromArgv([dirn]):
pass
stream.seek(0)
buf = stream.read()
self.isin('...cell API (telepath): tcp://127.0.0.1:0', buf)
self.isin('...cell API (https): 0', buf)
conf = {
'dmon:listen': None,
'https:port': None,
}
s_common.yamlsave(conf, dirn, 'cell.yaml')
with self.getAsyncLoggerStream('synapse.lib.cell') as stream:
async with await s_cell.Cell.initFromArgv([dirn]):
pass
stream.seek(0)
buf = stream.read()
self.isin(f'...cell API (telepath): cell://root@{dirn}:*', buf)
self.isin('...cell API (https): disabled', buf)
async def test_stormlib_backup(self):
with self.getTestDir() as dirn:
backdirn = s_common.gendir(dirn, 'backup')
coredirn = s_common.gendir(dirn, 'cortex')
s_common.yamlsave({'backup:dir': backdirn}, coredirn, 'cell.yaml')
async with self.getTestCore(dirn=coredirn) as core:
self.eq((), await core.callStorm('return($lib.backup.list())'))
name = await core.callStorm('return($lib.backup.run())')
self.true(os.path.isdir(os.path.join(backdirn, name)))
self.eq((name, ), await
core.callStorm('return($lib.backup.list())'))
await core.callStorm('$lib.backup.del($name)',
opts={'vars': {
'name': name
}})
self.false(os.path.isdir(os.path.join(backdirn, name)))
await core.callStorm('$lib.backup.run(name=foo)',
opts={'vars': {
'name': name
}})
self.true(os.path.isdir(os.path.join(backdirn, 'foo')))
await core.callStorm('$lib.backup.del(foo)')
self.false(os.path.isdir(os.path.join(backdirn, 'foo')))
async def test_telepath_loadenv(self):
with self.getTestDir() as dirn:
certpath = s_common.gendir(dirn, 'certs')
newppath = s_common.genpath(dirn, 'newps')
conf = {
'version': 1,
'aha:servers': [
'tcp://localhost:9999/',
],
'certdirs': [
certpath,
newppath,
],
}
path = s_common.genpath(dirn, 'telepath.yaml')
s_common.yamlsave(conf, path)
fini = await s_telepath.loadTeleEnv(path)
await fini()
self.none(await s_telepath.loadTeleEnv(newppath))
conf['version'] = 99
s_common.yamlsave(conf, path)
self.none(await s_telepath.loadTeleEnv(path))
async def test_cell_backup(self):
async with self.getTestCore() as core:
with self.raises(s_exc.NeedConfValu):
await core.runBackup()
with self.raises(s_exc.NeedConfValu):
await core.getBackups()
with self.raises(s_exc.NeedConfValu):
await core.delBackup('foo')
with self.getTestDir() as dirn:
s_common.yamlsave({'backup:dir': dirn}, dirn, 'cell.yaml')
with self.raises(s_exc.BadConfValu):
async with self.getTestCore(dirn=dirn) as core:
pass
with self.getTestDir() as dirn:
backdirn = os.path.join(dirn, 'backups')
coredirn = os.path.join(dirn, 'cortex')
conf = {'backup:dir': backdirn}
s_common.yamlsave(conf, coredirn, 'cell.yaml')
async with self.getTestCore(dirn=coredirn) as core:
async with core.getLocalProxy() as proxy:
with self.raises(s_exc.BadArg):
await proxy.runBackup('../woot')
with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 0.1):
with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeperProc)):
await self.asyncraises(s_exc.SynErr, proxy.runBackup())
# Test runners can take an unusually long time to spawn a process
with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 8.0):
with mock.patch.object(s_cell.Cell, 'BACKUP_ACQUIRE_TIMEOUT', 0.1):
with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeper2Proc)):
await self.asyncraises(s_exc.SynErr, proxy.runBackup())
with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_exiterProc)):
await self.asyncraises(s_exc.SpawnExit, proxy.runBackup())
name = await proxy.runBackup()
self.eq((name,), await proxy.getBackups())
await proxy.delBackup(name)
self.eq((), await proxy.getBackups())
name = await proxy.runBackup(name='foo/bar')
with self.raises(s_exc.BadArg):
await proxy.delBackup(name='foo')
self.true(os.path.isdir(os.path.join(backdirn, 'foo', 'bar')))
self.eq(('foo/bar',), await proxy.getBackups())
with self.raises(s_exc.BadArg):
await proxy.runBackup(name='foo/bar')
def getTestConfDir(self, name, boot=None, conf=None):
with self.getTestDir() as dirn:
cdir = os.path.join(dirn, name)
s_common.makedirs(cdir)
if boot:
s_common.yamlsave(boot, cdir, 'boot.yaml')
if conf:
s_common.yamlsave(conf, cdir, 'cell.yaml')
yield dirn
def getTestConfDir(self, name, boot=None, conf=None):
with self.getTestDir() as dirn:
cdir = os.path.join(dirn, name)
s_common.makedirs(cdir)
if boot:
s_common.yamlsave(boot, cdir, 'boot.yaml')
if conf:
s_common.yamlsave(conf, cdir, 'cell.yaml')
yield dirn
async def test_servers_stemcell(self):
with self.getTestDir() as dirn:
cellyaml = os.path.join(dirn, 'cell.yaml')
conf = {'cell:ctor': 'synapse.cells.axon'}
s_common.yamlsave(conf, cellyaml)
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_axon.Axon)
conf = {'cell:ctor': 'synapse.cells.cortex'}
s_common.yamlsave(conf, cellyaml)
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_cortex.Cortex)
conf = {'cell:ctor': 'synapse.cells.jsonstor'}
s_common.yamlsave(conf, cellyaml)
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_jsonstor.JsonStorCell)
conf = {'cell:ctor': 'synapse.cells.aha'}
s_common.yamlsave(conf, cellyaml)
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_aha.AhaCell)
# Direct python class paths
conf = {'cell:ctor': 'synapse.lib.cell.Cell'}
s_common.yamlsave(conf, cellyaml)
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_cell.Cell)
# Resolve a envar
os.unlink(cellyaml)
self.false(os.path.isfile(cellyaml))
with self.setTstEnvars(
SYN_STEM_CELL_CTOR='synapse.cells.cryotank'):
cell = s_stemcell.getStemCell(dirn)
self.true(cell is s_cryotank.CryoCell)
# Sad paths
with self.setTstEnvars(SYN_STEM_CELL_CTOR='synapse.lib.newp.Newp'):
with self.raises(s_exc.NoSuchCtor):
cell = s_stemcell.getStemCell(dirn)
with self.raises(s_exc.NoSuchFile):
cell = s_stemcell.getStemCell(dirn)
os.rmdir(dirn)
with self.raises(s_exc.NoSuchDir):
cell = s_stemcell.getStemCell(dirn)
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargpaser()
opts = pars.parse_args(argv)
if opts.module:
mod = s_dyndeps.tryDynMod(opts.module)
outp.printf(f'Loaded {opts.module}@{mod}')
if opts.cells:
outp.printf('Registered cells:')
for cname, cpath in s_cells.getCells():
outp.printf(f'{cname:<10} {cpath:>10}')
return 0
dirn = s_common.genpath(opts.dmonpath, 'cells', opts.cellname)
if os.path.isdir(dirn):
outp.printf(f'cell directory already exists: {dirn}')
return 1
dmon = {}
if opts.listen:
dmon['listen'] = opts.listen
if opts.module:
dmon['modules'] = [opts.module]
if dmon:
dmon.setdefault('modules', [])
dmon_fp = os.path.join(opts.dmonpath, 'dmon.yaml')
if os.path.exists(dmon_fp):
outp.printf(f'Cannot overwrite existing dmon.yaml file. [{dmon_fp}]')
return 1
s_common.yamlsave(dmon, dmon_fp)
boot = {
'cell:name': opts.cellname,
}
if opts.auth:
boot['auth:en'] = True
if opts.admin:
boot['auth:en'] = True
boot['auth:admin'] = opts.admin
outp.printf(f'Deploying a {opts.celltype} at: {dirn}')
s_cells.deploy(opts.celltype, dirn, boot)
return 0
async def test_storm_file_optfile(self):
async with self.getTestCoreAndProxy() as (core, prox):
test_opts = {'vars': {'hehe': 'woot.com'}}
dirn = s_common.gendir(core.dirn, 'junk')
optsfile = os.path.join(dirn, 'woot.json')
optsfile_yaml = os.path.join(dirn, 'woot.yaml')
stormfile = os.path.join(dirn, 'woot.storm')
with s_common.genfile(stormfile) as fd:
fd.write(b'[ inet:fqdn=$hehe ]')
s_common.jssave(test_opts, optsfile)
s_common.yamlsave(test_opts, optsfile_yaml)
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
await cmdr.runCmdLine(
f'storm --optsfile {optsfile} --file {stormfile}')
self.true(outp.expect('inet:fqdn=woot.com'))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
await cmdr.runCmdLine(
f'storm --optsfile {optsfile_yaml} --file {stormfile}')
self.true(outp.expect('inet:fqdn=woot.com'))
# Sad path cases
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
await cmdr.runCmdLine(
f'storm --file {stormfile} --optsfile {optsfile} .created')
self.true(
outp.expect(
'Cannot use a storm file and manual query together.'))
self.false(outp.expect('inet:fqdn=woot.com', throw=False))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
await cmdr.runCmdLine(f'storm --file {stormfile} --optsfile newp')
self.true(outp.expect('optsfile not found'))
outp = self.getTestOutp()
cmdr = await s_cmdr.getItemCmdr(prox, outp=outp)
await cmdr.runCmdLine(f'storm --file newp --optsfile {optsfile}')
self.true(outp.expect('file not found'))
async def _makeDefaultLayer(self):
'''
Since a user hasn't specified any layers, make one
'''
import synapse.cells as s_cells
layerdir = s_common.gendir(self.dirn, 'layers', DEFAULT_LAYER_NAME)
s_cells.deploy('layer-lmdb', layerdir)
mapsize = self.conf.get('layer:lmdb:mapsize')
if mapsize is not None:
cell_yaml = pathlib.Path(layerdir, 'cell.yaml')
conf = s_common.yamlload(cell_yaml) or {}
conf['lmdb:mapsize'] = mapsize
s_common.yamlsave(conf, cell_yaml)
logger.info('Creating a new default storage layer at %s', layerdir)
return await s_cells.initFromDirn(layerdir)
async def test_cell_backup(self):
async with self.getTestCore() as core:
with self.raises(s_exc.NeedConfValu):
await core.runBackup()
with self.raises(s_exc.NeedConfValu):
await core.getBackups()
with self.raises(s_exc.NeedConfValu):
await core.delBackup('foo')
with self.getTestDir() as dirn:
s_common.yamlsave({'backup:dir': dirn}, dirn, 'cell.yaml')
with self.raises(s_exc.BadConfValu):
async with self.getTestCore(dirn=dirn) as core:
pass
with self.getTestDir() as dirn:
backdirn = os.path.join(dirn, 'backups')
coredirn = os.path.join(dirn, 'cortex')
conf = {'backup:dir': backdirn}
s_common.yamlsave(conf, coredirn, 'cell.yaml')
async with self.getTestCore(dirn=coredirn) as core:
async with core.getLocalProxy() as proxy:
with self.raises(s_exc.BadArg):
await proxy.runBackup('../woot')
name = await proxy.runBackup()
self.eq((name, ), await proxy.getBackups())
await proxy.delBackup(name)
self.eq((), await proxy.getBackups())
name = await proxy.runBackup(name='foo/bar')
with self.raises(s_exc.BadArg):
await proxy.delBackup(name='foo')
self.true(
os.path.isdir(os.path.join(backdirn, 'foo', 'bar')))
self.eq(('foo/bar', ), await proxy.getBackups())
with self.raises(s_exc.BadArg):
await proxy.runBackup(name='foo/bar')
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(
prog='synapse.tools.hive.save',
description='Save tree data from a remote hive to file.')
pars.add_argument('--path',
default=None,
help='A hive path string to use as the root.')
pars.add_argument(
'--yaml',
default=False,
action='store_true',
help='Parse the savefile as a YAML file (default: msgpack)')
pars.add_argument('hiveurl', help='The telepath URL for the remote hive.')
pars.add_argument('filepath', help='The local file path to save.')
opts = pars.parse_args(argv)
path = ()
if opts.path is not None:
path = opts.path.split('/')
async with await s_telepath.openurl(opts.hiveurl) as hive:
try:
s_version.reqVersion(hive._getSynVers(), reqver)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
outp.printf(
f'Hive version {valu} is outside of the hive.save supported range ({reqver}).'
)
outp.printf(
f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.'
)
return 1
tree = await hive.saveHiveTree(path=path)
if opts.yaml:
s_common.yamlsave(tree, opts.filepath)
else:
s_msgpack.dumpfile(tree, opts.filepath)
return 0
async def test_cellauth_passwd(self):
with self.getTestSynDir() as syndir:
s_common.yamlsave({'version': 1}, syndir, 'telepath.yaml')
async with self.getTestCore() as core:
coreurl = core.getLocalUrl()
outp = self.getTestOutp()
argv = [coreurl, 'modify', '--adduser', 'foo']
await s_cellauth.main(argv, outp)
outp = self.getTestOutp()
argv = [coreurl, 'modify', '--passwd', 'mysecret', 'foo']
await s_cellauth.main(argv, outp)
outp.expect('setting passwd for: foo')
def deploy(name, dirn, boot=None):
'''
Deploy a cell of the named type to the specified directory.
'''
ctor = ctors.get(name)
if ctor is None:
raise s_exc.NoSuchName(name=name, mesg='No cell ctor by that name')
if boot is None:
boot = {}
boot['type'] = name
# create the boot.yaml
s_common.yamlsave(boot, dirn, 'boot.yaml')
# Cell has a deploy class method (possibly per cell type)
ctor.deploy(dirn)
async def test_stormlib_backup(self):
with self.getTestDir() as dirn:
backdirn = s_common.gendir(dirn, 'backup')
coredirn = s_common.gendir(dirn, 'cortex')
s_common.yamlsave({'backup:dir': backdirn}, coredirn, 'cell.yaml')
async with self.getTestCore(dirn=coredirn) as core:
self.eq((), await core.callStorm('return($lib.backup.list())'))
name = await core.callStorm('return($lib.backup.run())')
self.true(os.path.isdir(os.path.join(backdirn, name)))
self.eq((name, ), await
core.callStorm('return($lib.backup.list())'))
await core.callStorm('$lib.backup.del($name)',
opts={'vars': {
'name': name
}})
self.false(os.path.isdir(os.path.join(backdirn, name)))
await core.callStorm('$lib.backup.run(name=foo)',
opts={'vars': {
'name': name
}})
self.true(os.path.isdir(os.path.join(backdirn, 'foo')))
await core.callStorm('$lib.backup.del(foo)')
self.false(os.path.isdir(os.path.join(backdirn, 'foo')))
async def mockBackupTask(dirn):
await asyncio.sleep(5)
core._execBackupTask = mockBackupTask
with self.raises(s_exc.BackupAlreadyRunning):
q = 'return(($lib.backup.run(wait=(false)) + $lib.backup.run(wait=(false))))'
name = await core.callStorm(q)
async def test_cell_authpasswd(self):
conf = {
'auth:passwd': 'cottoncandy',
}
pconf = {'user': '******', 'passwd': 'cottoncandy'}
with self.getTestDir() as dirn:
s_common.yamlsave(conf, dirn, 'cell.yaml')
async with await EchoAuth.anit(dirn) as echo:
# start a regular network listener so we can auth
host, port = await echo.dmon.listen('tcp://127.0.0.1:0/')
async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/',
**pconf) as proxy:
self.true(await proxy.isadmin())
self.true(await proxy.allowed(('hehe', 'haha')))
url = f'tcp://[email protected]:{port}/'
await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url))
os.unlink(s_common.genpath(dirn, 'cell.yaml'))
# Pass the auth data in via conf directly
async with await EchoAuth.anit(dirn,
conf={'auth:passwd':
'pennywise'}) as echo:
# start a regular network listener so we can auth
host, port = await echo.dmon.listen('tcp://127.0.0.1:0/')
url = f'tcp://*****:*****@127.0.0.1:{port}/'
async with await s_telepath.openurl(url) as proxy:
self.true(await proxy.isadmin())
self.true(await proxy.allowed(('hehe', 'haha')))
# Ensure the cell and its auth have been fini'd
self.true(echo.isfini)
self.true(echo.auth.isfini)
root = await echo.auth.getUserByName('root')
self.true(root.isfini)
async def test_server_mirror(self):
with self.getTestDir() as dirn, self.withSetLoggingMock():
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
async with self.getTestCore(dirn=path00) as core00:
await core00.nodes('[ inet:ipv4=1.2.3.4 ]')
s_tools_backup.backup(path00, path01)
async with self.getTestCore(dirn=path00) as core00:
outp = self.getTestOutp()
argv = ['--telepath', 'tcp://127.0.0.1:0/',
'--https', '0',
'--mirror', core00.getLocalUrl(),
path01]
# add a node for core01 to sync before window
await core00.nodes('[ inet:ipv4=5.5.5.5 ]')
offs = core00.nexsroot.getOffset()
s_common.yamlsave({'logchanges': True}, path01, 'cell.yaml')
async with await s_cortex.Cortex.initFromArgv(argv, outp=outp) as core01:
# TODO functionalize this API on the cortex (cell?)
await core01.nexsroot.waitForOffset(offs - 1)
self.len(1, await core01.nodes('inet:ipv4=5.5.5.5'))
# add a node for core01 to sync via window
self.len(1, await core00.nodes('[ inet:ipv4=6.6.6.6 ]'))
offs = core00.nexsroot.getOffset()
await core01.nexsroot.waitForOffset(offs - 1)
self.len(1, await core01.nodes('inet:ipv4=6.6.6.6'))
async def test_server_mirror(self):
with self.getTestDir() as dirn:
async with self.withSetLoggingMock() as mock:
path00 = s_common.gendir(dirn, 'core00')
path01 = s_common.gendir(dirn, 'core01')
async with self.getTestCore(dirn=path00) as core00:
await core00.nodes('[ inet:ipv4=1.2.3.4 ]')
s_tools_backup.backup(path00, path01)
async with self.getTestCore(dirn=path00) as core00:
argv = [
'--telepath', 'tcp://127.0.0.1:0/', '--https', '0',
'--mirror',
core00.getLocalUrl(), path01
]
# add a node for core01 to sync before window
await core00.nodes('[ inet:ipv4=5.5.5.5 ]')
s_common.yamlsave({'nexslog:en': True}, path01,
'cell.yaml')
async with await s_cortex.Cortex.initFromArgv(argv
) as core01:
await core01.sync()
self.len(1, await core01.nodes('inet:ipv4=5.5.5.5'))
# add a node for core01 to sync via window
self.len(1, await
core00.nodes('[ inet:ipv4=6.6.6.6 ]'))
await core01.sync()
self.len(1, await core01.nodes('inet:ipv4=6.6.6.6'))
async def test_alias(self):
item = TeleAware()
name = 'item'
async with self.getTestDmon() as dmon:
host, port = dmon.addr
dmon.share(name, item)
with self.getTestDir() as dirn:
url = f'tcp://{host}:{port}/{name}'
beepbeep_alias = url + '/beepbeep'
aliases = {name: url, f'{name}/borp': beepbeep_alias}
with self.setSynDir(dirn):
fp = s_common.getSynPath('aliases.yaml')
s_common.yamlsave(aliases, fp)
# None existent aliases return None
self.none(s_telepath.alias('newp'))
self.none(s_telepath.alias('newp/path'))
# An exact match wins
self.eq(s_telepath.alias(name), url)
self.eq(s_telepath.alias(f'{name}/borp'), beepbeep_alias)
# Dynamic aliases are valid.
self.eq(s_telepath.alias(f'{name}/beepbeep'),
beepbeep_alias)
async with await s_telepath.openurl(name) as prox:
self.eq(10, await prox.getFooBar(20, 10))
# Check to see that we can connect to an aliased name
# with a dynamic share attached to it.
async with await s_telepath.openurl(f'{name}/bar') as prox:
self.eq('bar: beep', await prox.beep())
