async def test_syningest_remote(self): async with self.getTestCore() as core: guid = s_common.guid() seen = s_common.now() gestdef = self.getIngestDef(guid, seen) with self.getTestDir() as dirn: # Test yaml support here gestfp = s_common.genpath(dirn, 'gest.yaml') s_common.yamlsave(gestdef, gestfp) argv = ['--cortex', core.getLocalUrl(), '--debug', '--modules', 'synapse.tests.utils.TestModule', gestfp] outp = self.getTestOutp() cmdg = s_t_utils.CmdGenerator(['storm test:pivcomp -> *', EOFError()]) with self.withCliPromptMockExtendOutp(outp): with self.withTestCmdr(cmdg): self.eq(await s_feed.main(argv, outp=outp), 0) self.true(outp.expect('test:str=haha', throw=False)) self.true(outp.expect('test:pivtarg=hehe', throw=False))
async def test_cell_nonstandard_admin(self): boot = { 'auth:admin': 'pennywise:cottoncandy', 'type': 'echoauth', } pconf = {'user': '******', 'passwd': 'cottoncandy'} with self.getTestDir('cellauth') as dirn: s_common.yamlsave(boot, dirn, 'boot.yaml') async with await EchoAuth.anit(dirn) as echo: echo.insecure = False # start a regular network listener so we can auth host, port = await echo.dmon.listen('tcp://127.0.0.1:0/') async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/', **pconf) as proxy: self.true(await proxy.isadmin()) self.true(await proxy.allowed(('hehe', 'haha'))) url = f'tcp://[email protected]:{port}/' await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url)) # Ensure the cell and its auth have been fini'd self.true(echo.isfini) self.true(echo.auth.isfini) self.true(echo.auth.getUserByName('root').isfini) self.true(echo.auth.getUserByName('pennywise').isfini)
def testmain(): pconf = {'user': '******', 'passwd': 'root'} with self.getTestProxy(dmon, 'core', **pconf) as core: # Setup user permissions core.addAuthRole('creator') core.addAuthRule('creator', (True, ('node:add', ))) core.addAuthRule('creator', (True, ('prop:set', ))) core.addAuthRule('creator', (True, ('tag:add', ))) core.addUserRole('root', 'creator') host, port = dmon.addr curl = f'tcp://*****:*****@{host}:{port}/core' guid = s_common.guid() seen = s_common.now() gestdef = self.getIngestDef(guid, seen) with self.getTestDir() as dirn: # Test yaml support here gestfp = s_common.genpath(dirn, 'gest.yaml') s_common.yamlsave(gestdef, gestfp) argv = [ '--cortex', curl, '--debug', '--modules', 'synapse.tests.utils.TestModule', gestfp ] outp = self.getTestOutp() cmdg = s_t_utils.CmdGenerator(['storm pivcomp -> *'], on_end=EOFError) with mock.patch('synapse.lib.cli.get_input', cmdg): self.eq(s_feed.main(argv, outp=outp), 0) self.true(outp.expect('teststr=haha', throw=False)) self.true(outp.expect('pivtarg=hehe', throw=False))
async def test_load_failures(self): async with self.getTestCore() as core: # type: s_cortex.Cortex with self.setTstEnvars(SYN_TEST_MOD_FAIL_PRE=1) as cm: with self.getAsyncLoggerStream('synapse.cortex', 'preCoreModuleFail') as stream: self.none(await core.loadCoreModule(foo_ctor)) self.true(await stream.wait(1)) self.none(core.getCoreMod(foo_ctor)) with self.setTstEnvars(SYN_TEST_MOD_FAIL_INIT=1) as cm: with self.getAsyncLoggerStream('synapse.cortex', 'initCoreModuleFail') as stream: self.none(await core.loadCoreModule(foo_ctor)) self.true(await stream.wait(1)) self.none(core.getCoreMod(foo_ctor)) with self.getTestDir(mirror='testcore') as dirn: conf = s_common.yamlload(dirn, 'cell.yaml') conf['modules'].append(foo_ctor) s_common.yamlsave(conf, dirn, 'cell.yaml') conf = s_common.yamlload(dirn, 'cell.yaml') with self.setTstEnvars(SYN_TEST_MOD_FAIL_PRE=1) as cm: with self.getAsyncLoggerStream('synapse.cortex', 'preCoreModuleFail') as stream: async with await s_cortex.Cortex.anit(dirn) as core: self.true(await stream.wait(1)) self.none(core.getCoreMod(foo_ctor)) with self.setTstEnvars(SYN_TEST_MOD_FAIL_INIT=1) as cm: with self.getAsyncLoggerStream('synapse.cortex', 'initCoreModuleFail') as stream: async with await s_cortex.Cortex.anit(dirn) as core: self.true(await stream.wait(1)) self.none(core.getCoreMod(foo_ctor))
def test_common_yaml(self): obj = [{'key': 1, 'key2': [1, 2, 3], 'key3': True, 'key4': 'some str', 'key5': { 'oh': 'my', 'we all': 'float down here' }, }, 'duck', False, 'zero', 0.1, ] with self.getTestDir() as dirn: s_common.yamlsave(obj, dirn, 'test.yaml') robj = s_common.yamlload(dirn, 'test.yaml') self.eq(obj, robj) obj = {'foo': 'bar', 'zap': [3, 4, 'f']} s_common.yamlsave(obj, dirn, 'test.yaml') s_common.yamlmod({'bar': 42}, dirn, 'test.yaml') robj = s_common.yamlload(dirn, 'test.yaml') obj['bar'] = 42 self.eq(obj, robj) # Test yaml helper safety s = '!!python/object/apply:os.system ["pwd"]' with s_common.genfile(dirn, 'explode.yaml') as fd: fd.write(s.encode()) self.raises(yaml.YAMLError, s_common.yamlload, dirn, 'explode.yaml')
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.save', description='Save tree data from a remote hive to file.') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to save.') opts = pars.parse_args(argv) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: tree = await hive.saveHiveTree(path=path) if opts.yaml: s_common.yamlsave(tree, opts.filepath) else: s_msgpack.dumpfile(tree, opts.filepath)
async def test_cell_hiveboot(self): with self.getTestDir() as dirn: tree = { 'kids': { 'hehe': { 'value': 'haha' }, } } bootpath = os.path.join(dirn, 'hiveboot.yaml') s_common.yamlsave(tree, bootpath) async with await s_cell.Cell.anit(dirn) as cell: self.eq('haha', await cell.hive.get(('hehe', ))) # test that the file does not load again tree['kids']['redbaloons'] = {'value': 99} s_common.yamlsave(tree, bootpath) async with await s_cell.Cell.anit(dirn) as cell: self.none(await cell.hive.get(('redbaloons', )))
def test_doc_data(self): with self.getTestDir() as dirn: s_common.gendir(dirn, 'docdata', 'stuff') docdata = s_common.genpath(dirn, 'docdata') root = s_common.genpath(dirn, 'synapse', 'userguides') d = {'key': 'value'} s_common.jssave(d, docdata, 'data.json') s_common.yamlsave(d, docdata, 'data.yaml') s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk')) with s_common.genfile(docdata, 'stuff', 'data.txt') as fd: fd.write('beep'.encode()) with s_common.genfile(docdata, 'data.jsonl') as fd: fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') data = s_jupyter.getDocData('data.json', root) self.eq(data, d) data = s_jupyter.getDocData('data.yaml', root) self.eq(data, d) data = s_jupyter.getDocData('data.mpk', root) self.eq(data, d) data = s_jupyter.getDocData('stuff/data.txt', root) self.eq(data, b'beep') data = s_jupyter.getDocData('data.jsonl', root) self.eq(data, [d, d, d]) self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root) self.raises(ValueError, s_jupyter.getDocData, '../../../../../../etc/passwd', root)
async def test_cell_nonstandard_admin(self): boot = { 'auth:admin': 'pennywise:cottoncandy', 'type': 'echoauth', } pconf = {'user': '******', 'passwd': 'cottoncandy'} with self.getTestDir('cellauth') as dirn: s_common.yamlsave(boot, dirn, 'boot.yaml') async with await EchoAuth.anit(dirn) as echo: echo.insecure = False # start a regular network listener so we can auth host, port = await echo.dmon.listen('tcp://127.0.0.1:0/') async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/', **pconf) as proxy: self.true(await proxy.isadmin()) self.true(await proxy.allowed('hehe', 'haha')) url = f'tcp://[email protected]:{port}/' await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url)) # Ensure the cell and its auth have been fini'd self.true(echo.isfini) self.true(echo.auth.isfini) self.true(echo.auth.getUserByName('root').isfini) self.true(echo.auth.getUserByName('pennywise').isfini)
async def test_server_mirror_badiden(self): with self.getTestDir() as dirn, self.withSetLoggingMock(): path00 = s_common.gendir(dirn, 'core00') path01 = s_common.gendir(dirn, 'core01') argv0 = ['--telepath', 'tcp://127.0.0.1:0/', '--https', '0', '--name', 'srccore', path00, ] out0 = self.getTestOutp() async with await s_cortex.Cortex.initFromArgv(argv0, outp=out0) as core00: out1 = self.getTestOutp() argv1 = ['--telepath', 'tcp://127.0.0.1:0/', '--https', '0', '--mirror', core00.getLocalUrl(), path01, ] s_common.yamlsave({'logchanges': True}, path01, 'cell.yaml') async with await s_cortex.Cortex.initFromArgv(argv1, outp=out1) as core01: await core01.waitfini(6) self.true(core01.isfini)
async def test_basics(self): async with self.getTestCore() as core: # type: s_cortex.Cortex testmod = core.getCoreMod('synapse.tests.utils.TestModule') self.isinstance(testmod, s_module.CoreModule) # modname from class name self.eq(testmod.mod_name, 'testmodule') foomod = await core.loadCoreModule(foo_ctor) # modname from explicit modname self.eq(foomod.mod_name, 'foo') # modpaths are dynamically made on demand self.false(os.path.isdir(foomod._modpath)) mpath = foomod.getModPath() self.isin(os.path.join('mods', 'foo'), mpath) self.true(os.path.isdir(foomod._modpath)) # preload a config file for the BarModule dirn = s_common.gendir(core.dirn, 'mods', 'barmod') s_common.yamlsave({'test': 1, 'duck': 'quack'}, dirn, 'conf.yaml') # barmodule loads a layerctor self.false(core.layrctors.get('newp') is int) barmod = await core.loadCoreModule(bar_ctor) self.eq(barmod.data, {}) self.eq(barmod.conf, { 'test': 1, 'hehe': 'haha', 'duck': 'quack', }) self.true(core.layrctors.get('newp') is int) self.eq(barmod.data, {'fini': True})
def test_doc_data(self): with self.getTestDir() as dirn: s_common.gendir(dirn, 'docdata', 'stuff') docdata = s_common.genpath(dirn, 'docdata') root = s_common.genpath(dirn, 'synapse', 'userguides') d = {'key': 'value'} s_common.jssave(d, docdata, 'data.json') s_common.yamlsave(d, docdata, 'data.yaml') s_msgpack.dumpfile(d, os.path.join(docdata, 'data.mpk')) with s_common.genfile(docdata, 'stuff', 'data.txt') as fd: fd.write('beep'.encode()) with s_common.genfile(docdata, 'data.jsonl') as fd: fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') fd.write(json.dumps(d).encode() + b'\n') data = s_jupyter.getDocData('data.json', root) self.eq(data, d) data = s_jupyter.getDocData('data.yaml', root) self.eq(data, d) data = s_jupyter.getDocData('data.mpk', root) self.eq(data, d) data = s_jupyter.getDocData('stuff/data.txt', root) self.eq(data, b'beep') data = s_jupyter.getDocData('data.jsonl', root) self.eq(data, [d, d, d]) self.raises(ValueError, s_jupyter.getDocData, 'newp.bin', root) self.raises(ValueError, s_jupyter.getDocData, '../../../../../../etc/passwd', root)
async def test_cell_confprint(self): with self.withSetLoggingMock(): with self.getTestDir() as dirn: conf = { 'dmon:listen': 'tcp://127.0.0.1:0', 'https:port': 0, } s_common.yamlsave(conf, dirn, 'cell.yaml') outp = self.getTestOutp() async with await s_cell.Cell.initFromArgv([dirn], outp=outp) as cell: outp.expect('...cell API (telepath): tcp://127.0.0.1:0') outp.expect('...cell API (https): 0') conf = { 'dmon:listen': 'tcp://127.0.0.1:0', 'https:port': None, } s_common.yamlsave(conf, dirn, 'cell.yaml') outp = self.getTestOutp() async with await s_cell.Cell.initFromArgv([dirn], outp=outp) as cell: outp.expect('...cell API (telepath): tcp://127.0.0.1:0') outp.expect('...cell API (https): disabled')
async def test_lib_aha_loadenv(self): with self.getTestDir() as dirn: async with self.getTestAha() as aha: host, port = await aha.dmon.listen('tcp://127.0.0.1:0') await aha.auth.rootuser.setPasswd('hehehaha') conf = { 'version': 1, 'aha:servers': [ f'tcp://*****:*****@127.0.0.1:{port}/', ], } path = s_common.genpath(dirn, 'telepath.yaml') s_common.yamlsave(conf, path) fini = await s_telepath.loadTeleEnv(path) # Should be one uninitialized aha client self.len(1, s_telepath.aha_clients) [info] = s_telepath.aha_clients.values() self.none(info.get('client')) with self.raises(s_exc.NoSuchName): await s_telepath.openurl('aha://[email protected]') # Connecting to an aha url should have initialized the client self.len(1, s_telepath.aha_clients) self.nn(info.get('client')) await fini()
async def test_migr_yamlmod(self): with self.getRegrDir('cortexes', REGR_VER) as src: with self.getTestDir() as dest: locallyrs = os.listdir(os.path.join(src, 'layers')) mods = {'modules': ['synapse.tests.utils.TestModule']} s_common.yamlsave(mods, src, 'cell.yaml') conf = { 'src': src, 'dest': dest, 'migrops': ['dmodel', 'hivelyr', 'nodes'], 'editbatchsize': 1, 'nodelim': 2, } async with await s_migr.Migrator.anit(conf) as migr: # test that we can skip dirn migration if it already exists migr.migrops.append('dirn') await migr._migrDirn() del migr.migrops[-1] # verify that test:int was loaded and migrated await migr.migrate() stats = [] for iden in locallyrs: statkey = f'{iden}:form:test:int' stats.extend([log async for log in migr._migrlogGet('nodes', 'stat', statkey)]) self.eq((1, 1), stats[0]['val'])
async def test_cell_confprint(self): with self.withSetLoggingMock(): with self.getTestDir() as dirn: conf = { 'dmon:listen': 'tcp://127.0.0.1:0', 'https:port': 0, } s_common.yamlsave(conf, dirn, 'cell.yaml') with self.getAsyncLoggerStream('synapse.lib.cell') as stream: async with await s_cell.Cell.initFromArgv([dirn]): pass stream.seek(0) buf = stream.read() self.isin('...cell API (telepath): tcp://127.0.0.1:0', buf) self.isin('...cell API (https): 0', buf) conf = { 'dmon:listen': None, 'https:port': None, } s_common.yamlsave(conf, dirn, 'cell.yaml') with self.getAsyncLoggerStream('synapse.lib.cell') as stream: async with await s_cell.Cell.initFromArgv([dirn]): pass stream.seek(0) buf = stream.read() self.isin(f'...cell API (telepath): cell://root@{dirn}:*', buf) self.isin('...cell API (https): disabled', buf)
async def test_stormlib_backup(self): with self.getTestDir() as dirn: backdirn = s_common.gendir(dirn, 'backup') coredirn = s_common.gendir(dirn, 'cortex') s_common.yamlsave({'backup:dir': backdirn}, coredirn, 'cell.yaml') async with self.getTestCore(dirn=coredirn) as core: self.eq((), await core.callStorm('return($lib.backup.list())')) name = await core.callStorm('return($lib.backup.run())') self.true(os.path.isdir(os.path.join(backdirn, name))) self.eq((name, ), await core.callStorm('return($lib.backup.list())')) await core.callStorm('$lib.backup.del($name)', opts={'vars': { 'name': name }}) self.false(os.path.isdir(os.path.join(backdirn, name))) await core.callStorm('$lib.backup.run(name=foo)', opts={'vars': { 'name': name }}) self.true(os.path.isdir(os.path.join(backdirn, 'foo'))) await core.callStorm('$lib.backup.del(foo)') self.false(os.path.isdir(os.path.join(backdirn, 'foo')))
async def test_telepath_loadenv(self): with self.getTestDir() as dirn: certpath = s_common.gendir(dirn, 'certs') newppath = s_common.genpath(dirn, 'newps') conf = { 'version': 1, 'aha:servers': [ 'tcp://localhost:9999/', ], 'certdirs': [ certpath, newppath, ], } path = s_common.genpath(dirn, 'telepath.yaml') s_common.yamlsave(conf, path) fini = await s_telepath.loadTeleEnv(path) await fini() self.none(await s_telepath.loadTeleEnv(newppath)) conf['version'] = 99 s_common.yamlsave(conf, path) self.none(await s_telepath.loadTeleEnv(path))
async def test_cell_backup(self): async with self.getTestCore() as core: with self.raises(s_exc.NeedConfValu): await core.runBackup() with self.raises(s_exc.NeedConfValu): await core.getBackups() with self.raises(s_exc.NeedConfValu): await core.delBackup('foo') with self.getTestDir() as dirn: s_common.yamlsave({'backup:dir': dirn}, dirn, 'cell.yaml') with self.raises(s_exc.BadConfValu): async with self.getTestCore(dirn=dirn) as core: pass with self.getTestDir() as dirn: backdirn = os.path.join(dirn, 'backups') coredirn = os.path.join(dirn, 'cortex') conf = {'backup:dir': backdirn} s_common.yamlsave(conf, coredirn, 'cell.yaml') async with self.getTestCore(dirn=coredirn) as core: async with core.getLocalProxy() as proxy: with self.raises(s_exc.BadArg): await proxy.runBackup('../woot') with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 0.1): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeperProc)): await self.asyncraises(s_exc.SynErr, proxy.runBackup()) # Test runners can take an unusually long time to spawn a process with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 8.0): with mock.patch.object(s_cell.Cell, 'BACKUP_ACQUIRE_TIMEOUT', 0.1): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeper2Proc)): await self.asyncraises(s_exc.SynErr, proxy.runBackup()) with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_exiterProc)): await self.asyncraises(s_exc.SpawnExit, proxy.runBackup()) name = await proxy.runBackup() self.eq((name,), await proxy.getBackups()) await proxy.delBackup(name) self.eq((), await proxy.getBackups()) name = await proxy.runBackup(name='foo/bar') with self.raises(s_exc.BadArg): await proxy.delBackup(name='foo') self.true(os.path.isdir(os.path.join(backdirn, 'foo', 'bar'))) self.eq(('foo/bar',), await proxy.getBackups()) with self.raises(s_exc.BadArg): await proxy.runBackup(name='foo/bar')
def getTestConfDir(self, name, boot=None, conf=None): with self.getTestDir() as dirn: cdir = os.path.join(dirn, name) s_common.makedirs(cdir) if boot: s_common.yamlsave(boot, cdir, 'boot.yaml') if conf: s_common.yamlsave(conf, cdir, 'cell.yaml') yield dirn
def getTestConfDir(self, name, boot=None, conf=None): with self.getTestDir() as dirn: cdir = os.path.join(dirn, name) s_common.makedirs(cdir) if boot: s_common.yamlsave(boot, cdir, 'boot.yaml') if conf: s_common.yamlsave(conf, cdir, 'cell.yaml') yield dirn
async def test_servers_stemcell(self): with self.getTestDir() as dirn: cellyaml = os.path.join(dirn, 'cell.yaml') conf = {'cell:ctor': 'synapse.cells.axon'} s_common.yamlsave(conf, cellyaml) cell = s_stemcell.getStemCell(dirn) self.true(cell is s_axon.Axon) conf = {'cell:ctor': 'synapse.cells.cortex'} s_common.yamlsave(conf, cellyaml) cell = s_stemcell.getStemCell(dirn) self.true(cell is s_cortex.Cortex) conf = {'cell:ctor': 'synapse.cells.jsonstor'} s_common.yamlsave(conf, cellyaml) cell = s_stemcell.getStemCell(dirn) self.true(cell is s_jsonstor.JsonStorCell) conf = {'cell:ctor': 'synapse.cells.aha'} s_common.yamlsave(conf, cellyaml) cell = s_stemcell.getStemCell(dirn) self.true(cell is s_aha.AhaCell) # Direct python class paths conf = {'cell:ctor': 'synapse.lib.cell.Cell'} s_common.yamlsave(conf, cellyaml) cell = s_stemcell.getStemCell(dirn) self.true(cell is s_cell.Cell) # Resolve a envar os.unlink(cellyaml) self.false(os.path.isfile(cellyaml)) with self.setTstEnvars( SYN_STEM_CELL_CTOR='synapse.cells.cryotank'): cell = s_stemcell.getStemCell(dirn) self.true(cell is s_cryotank.CryoCell) # Sad paths with self.setTstEnvars(SYN_STEM_CELL_CTOR='synapse.lib.newp.Newp'): with self.raises(s_exc.NoSuchCtor): cell = s_stemcell.getStemCell(dirn) with self.raises(s_exc.NoSuchFile): cell = s_stemcell.getStemCell(dirn) os.rmdir(dirn) with self.raises(s_exc.NoSuchDir): cell = s_stemcell.getStemCell(dirn)
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargpaser() opts = pars.parse_args(argv) if opts.module: mod = s_dyndeps.tryDynMod(opts.module) outp.printf(f'Loaded {opts.module}@{mod}') if opts.cells: outp.printf('Registered cells:') for cname, cpath in s_cells.getCells(): outp.printf(f'{cname:<10} {cpath:>10}') return 0 dirn = s_common.genpath(opts.dmonpath, 'cells', opts.cellname) if os.path.isdir(dirn): outp.printf(f'cell directory already exists: {dirn}') return 1 dmon = {} if opts.listen: dmon['listen'] = opts.listen if opts.module: dmon['modules'] = [opts.module] if dmon: dmon.setdefault('modules', []) dmon_fp = os.path.join(opts.dmonpath, 'dmon.yaml') if os.path.exists(dmon_fp): outp.printf(f'Cannot overwrite existing dmon.yaml file. [{dmon_fp}]') return 1 s_common.yamlsave(dmon, dmon_fp) boot = { 'cell:name': opts.cellname, } if opts.auth: boot['auth:en'] = True if opts.admin: boot['auth:en'] = True boot['auth:admin'] = opts.admin outp.printf(f'Deploying a {opts.celltype} at: {dirn}') s_cells.deploy(opts.celltype, dirn, boot) return 0
async def test_storm_file_optfile(self): async with self.getTestCoreAndProxy() as (core, prox): test_opts = {'vars': {'hehe': 'woot.com'}} dirn = s_common.gendir(core.dirn, 'junk') optsfile = os.path.join(dirn, 'woot.json') optsfile_yaml = os.path.join(dirn, 'woot.yaml') stormfile = os.path.join(dirn, 'woot.storm') with s_common.genfile(stormfile) as fd: fd.write(b'[ inet:fqdn=$hehe ]') s_common.jssave(test_opts, optsfile) s_common.yamlsave(test_opts, optsfile_yaml) outp = self.getTestOutp() cmdr = await s_cmdr.getItemCmdr(prox, outp=outp) await cmdr.runCmdLine( f'storm --optsfile {optsfile} --file {stormfile}') self.true(outp.expect('inet:fqdn=woot.com')) outp = self.getTestOutp() cmdr = await s_cmdr.getItemCmdr(prox, outp=outp) await cmdr.runCmdLine( f'storm --optsfile {optsfile_yaml} --file {stormfile}') self.true(outp.expect('inet:fqdn=woot.com')) # Sad path cases outp = self.getTestOutp() cmdr = await s_cmdr.getItemCmdr(prox, outp=outp) await cmdr.runCmdLine( f'storm --file {stormfile} --optsfile {optsfile} .created') self.true( outp.expect( 'Cannot use a storm file and manual query together.')) self.false(outp.expect('inet:fqdn=woot.com', throw=False)) outp = self.getTestOutp() cmdr = await s_cmdr.getItemCmdr(prox, outp=outp) await cmdr.runCmdLine(f'storm --file {stormfile} --optsfile newp') self.true(outp.expect('optsfile not found')) outp = self.getTestOutp() cmdr = await s_cmdr.getItemCmdr(prox, outp=outp) await cmdr.runCmdLine(f'storm --file newp --optsfile {optsfile}') self.true(outp.expect('file not found'))
async def _makeDefaultLayer(self): ''' Since a user hasn't specified any layers, make one ''' import synapse.cells as s_cells layerdir = s_common.gendir(self.dirn, 'layers', DEFAULT_LAYER_NAME) s_cells.deploy('layer-lmdb', layerdir) mapsize = self.conf.get('layer:lmdb:mapsize') if mapsize is not None: cell_yaml = pathlib.Path(layerdir, 'cell.yaml') conf = s_common.yamlload(cell_yaml) or {} conf['lmdb:mapsize'] = mapsize s_common.yamlsave(conf, cell_yaml) logger.info('Creating a new default storage layer at %s', layerdir) return await s_cells.initFromDirn(layerdir)
async def test_cell_backup(self): async with self.getTestCore() as core: with self.raises(s_exc.NeedConfValu): await core.runBackup() with self.raises(s_exc.NeedConfValu): await core.getBackups() with self.raises(s_exc.NeedConfValu): await core.delBackup('foo') with self.getTestDir() as dirn: s_common.yamlsave({'backup:dir': dirn}, dirn, 'cell.yaml') with self.raises(s_exc.BadConfValu): async with self.getTestCore(dirn=dirn) as core: pass with self.getTestDir() as dirn: backdirn = os.path.join(dirn, 'backups') coredirn = os.path.join(dirn, 'cortex') conf = {'backup:dir': backdirn} s_common.yamlsave(conf, coredirn, 'cell.yaml') async with self.getTestCore(dirn=coredirn) as core: async with core.getLocalProxy() as proxy: with self.raises(s_exc.BadArg): await proxy.runBackup('../woot') name = await proxy.runBackup() self.eq((name, ), await proxy.getBackups()) await proxy.delBackup(name) self.eq((), await proxy.getBackups()) name = await proxy.runBackup(name='foo/bar') with self.raises(s_exc.BadArg): await proxy.delBackup(name='foo') self.true( os.path.isdir(os.path.join(backdirn, 'foo', 'bar'))) self.eq(('foo/bar', ), await proxy.getBackups()) with self.raises(s_exc.BadArg): await proxy.runBackup(name='foo/bar')
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser( prog='synapse.tools.hive.save', description='Save tree data from a remote hive to file.') pars.add_argument('--path', default=None, help='A hive path string to use as the root.') pars.add_argument( '--yaml', default=False, action='store_true', help='Parse the savefile as a YAML file (default: msgpack)') pars.add_argument('hiveurl', help='The telepath URL for the remote hive.') pars.add_argument('filepath', help='The local file path to save.') opts = pars.parse_args(argv) path = () if opts.path is not None: path = opts.path.split('/') async with await s_telepath.openurl(opts.hiveurl) as hive: try: s_version.reqVersion(hive._getSynVers(), reqver) except s_exc.BadVersion as e: valu = s_version.fmtVersion(*e.get('valu')) outp.printf( f'Hive version {valu} is outside of the hive.save supported range ({reqver}).' ) outp.printf( f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.' ) return 1 tree = await hive.saveHiveTree(path=path) if opts.yaml: s_common.yamlsave(tree, opts.filepath) else: s_msgpack.dumpfile(tree, opts.filepath) return 0
async def test_cellauth_passwd(self): with self.getTestSynDir() as syndir: s_common.yamlsave({'version': 1}, syndir, 'telepath.yaml') async with self.getTestCore() as core: coreurl = core.getLocalUrl() outp = self.getTestOutp() argv = [coreurl, 'modify', '--adduser', 'foo'] await s_cellauth.main(argv, outp) outp = self.getTestOutp() argv = [coreurl, 'modify', '--passwd', 'mysecret', 'foo'] await s_cellauth.main(argv, outp) outp.expect('setting passwd for: foo')
def deploy(name, dirn, boot=None): ''' Deploy a cell of the named type to the specified directory. ''' ctor = ctors.get(name) if ctor is None: raise s_exc.NoSuchName(name=name, mesg='No cell ctor by that name') if boot is None: boot = {} boot['type'] = name # create the boot.yaml s_common.yamlsave(boot, dirn, 'boot.yaml') # Cell has a deploy class method (possibly per cell type) ctor.deploy(dirn)
async def test_stormlib_backup(self): with self.getTestDir() as dirn: backdirn = s_common.gendir(dirn, 'backup') coredirn = s_common.gendir(dirn, 'cortex') s_common.yamlsave({'backup:dir': backdirn}, coredirn, 'cell.yaml') async with self.getTestCore(dirn=coredirn) as core: self.eq((), await core.callStorm('return($lib.backup.list())')) name = await core.callStorm('return($lib.backup.run())') self.true(os.path.isdir(os.path.join(backdirn, name))) self.eq((name, ), await core.callStorm('return($lib.backup.list())')) await core.callStorm('$lib.backup.del($name)', opts={'vars': { 'name': name }}) self.false(os.path.isdir(os.path.join(backdirn, name))) await core.callStorm('$lib.backup.run(name=foo)', opts={'vars': { 'name': name }}) self.true(os.path.isdir(os.path.join(backdirn, 'foo'))) await core.callStorm('$lib.backup.del(foo)') self.false(os.path.isdir(os.path.join(backdirn, 'foo'))) async def mockBackupTask(dirn): await asyncio.sleep(5) core._execBackupTask = mockBackupTask with self.raises(s_exc.BackupAlreadyRunning): q = 'return(($lib.backup.run(wait=(false)) + $lib.backup.run(wait=(false))))' name = await core.callStorm(q)
async def test_cell_authpasswd(self): conf = { 'auth:passwd': 'cottoncandy', } pconf = {'user': '******', 'passwd': 'cottoncandy'} with self.getTestDir() as dirn: s_common.yamlsave(conf, dirn, 'cell.yaml') async with await EchoAuth.anit(dirn) as echo: # start a regular network listener so we can auth host, port = await echo.dmon.listen('tcp://127.0.0.1:0/') async with await s_telepath.openurl(f'tcp://127.0.0.1:{port}/', **pconf) as proxy: self.true(await proxy.isadmin()) self.true(await proxy.allowed(('hehe', 'haha'))) url = f'tcp://[email protected]:{port}/' await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(url)) os.unlink(s_common.genpath(dirn, 'cell.yaml')) # Pass the auth data in via conf directly async with await EchoAuth.anit(dirn, conf={'auth:passwd': 'pennywise'}) as echo: # start a regular network listener so we can auth host, port = await echo.dmon.listen('tcp://127.0.0.1:0/') url = f'tcp://*****:*****@127.0.0.1:{port}/' async with await s_telepath.openurl(url) as proxy: self.true(await proxy.isadmin()) self.true(await proxy.allowed(('hehe', 'haha'))) # Ensure the cell and its auth have been fini'd self.true(echo.isfini) self.true(echo.auth.isfini) root = await echo.auth.getUserByName('root') self.true(root.isfini)
async def test_server_mirror(self): with self.getTestDir() as dirn, self.withSetLoggingMock(): path00 = s_common.gendir(dirn, 'core00') path01 = s_common.gendir(dirn, 'core01') async with self.getTestCore(dirn=path00) as core00: await core00.nodes('[ inet:ipv4=1.2.3.4 ]') s_tools_backup.backup(path00, path01) async with self.getTestCore(dirn=path00) as core00: outp = self.getTestOutp() argv = ['--telepath', 'tcp://127.0.0.1:0/', '--https', '0', '--mirror', core00.getLocalUrl(), path01] # add a node for core01 to sync before window await core00.nodes('[ inet:ipv4=5.5.5.5 ]') offs = core00.nexsroot.getOffset() s_common.yamlsave({'logchanges': True}, path01, 'cell.yaml') async with await s_cortex.Cortex.initFromArgv(argv, outp=outp) as core01: # TODO functionalize this API on the cortex (cell?) await core01.nexsroot.waitForOffset(offs - 1) self.len(1, await core01.nodes('inet:ipv4=5.5.5.5')) # add a node for core01 to sync via window self.len(1, await core00.nodes('[ inet:ipv4=6.6.6.6 ]')) offs = core00.nexsroot.getOffset() await core01.nexsroot.waitForOffset(offs - 1) self.len(1, await core01.nodes('inet:ipv4=6.6.6.6'))
async def test_server_mirror(self): with self.getTestDir() as dirn: async with self.withSetLoggingMock() as mock: path00 = s_common.gendir(dirn, 'core00') path01 = s_common.gendir(dirn, 'core01') async with self.getTestCore(dirn=path00) as core00: await core00.nodes('[ inet:ipv4=1.2.3.4 ]') s_tools_backup.backup(path00, path01) async with self.getTestCore(dirn=path00) as core00: argv = [ '--telepath', 'tcp://127.0.0.1:0/', '--https', '0', '--mirror', core00.getLocalUrl(), path01 ] # add a node for core01 to sync before window await core00.nodes('[ inet:ipv4=5.5.5.5 ]') s_common.yamlsave({'nexslog:en': True}, path01, 'cell.yaml') async with await s_cortex.Cortex.initFromArgv(argv ) as core01: await core01.sync() self.len(1, await core01.nodes('inet:ipv4=5.5.5.5')) # add a node for core01 to sync via window self.len(1, await core00.nodes('[ inet:ipv4=6.6.6.6 ]')) await core01.sync() self.len(1, await core01.nodes('inet:ipv4=6.6.6.6'))
async def test_alias(self): item = TeleAware() name = 'item' async with self.getTestDmon() as dmon: host, port = dmon.addr dmon.share(name, item) with self.getTestDir() as dirn: url = f'tcp://{host}:{port}/{name}' beepbeep_alias = url + '/beepbeep' aliases = {name: url, f'{name}/borp': beepbeep_alias} with self.setSynDir(dirn): fp = s_common.getSynPath('aliases.yaml') s_common.yamlsave(aliases, fp) # None existent aliases return None self.none(s_telepath.alias('newp')) self.none(s_telepath.alias('newp/path')) # An exact match wins self.eq(s_telepath.alias(name), url) self.eq(s_telepath.alias(f'{name}/borp'), beepbeep_alias) # Dynamic aliases are valid. self.eq(s_telepath.alias(f'{name}/beepbeep'), beepbeep_alias) async with await s_telepath.openurl(name) as prox: self.eq(10, await prox.getFooBar(20, 10)) # Check to see that we can connect to an aliased name # with a dynamic share attached to it. async with await s_telepath.openurl(f'{name}/bar') as prox: self.eq('bar: beep', await prox.beep())