async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
outp.printf('starting cryotank server: %s' % (opts.cryodir, ))
cryo = await s_cryotank.CryoCell.anit(opts.cryodir)
try:
outp.printf('...cryotank API (telepath): %s' % (opts.telepath, ))
await cryo.dmon.listen(opts.telepath)
outp.printf('...cryotank API (https): %s' % (opts.port, ))
await cryo.addHttpsPort(opts.port)
if opts.name:
outp.printf(f'...cryotank additional share name: {opts.name}')
cryo.dmon.share(opts.name, cryo)
return cryo
except Exception:
await cryo.fini()
raise
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
outp.printf('starting cryotank server: %s' % (opts.cryodir,))
cryo = await s_cryotank.CryoCell.anit(opts.cryodir)
try:
outp.printf('...cryotank API (telepath): %s' % (opts.telepath,))
await cryo.dmon.listen(opts.telepath)
outp.printf('...cryotank API (https): %s' % (opts.port,))
await cryo.addHttpsPort(opts.port)
if opts.name:
outp.printf(f'...cryotank additional share name: {opts.name}')
cryo.dmon.share(opts.name, cryo)
return cryo
except Exception:
await cryo.fini()
raise
async def main(argv): # pragma: no cover
if len(argv) not in (1, 2):
print(
'usage: python -m synapse.tools.cmdr <url> [<single quoted command>]'
)
return 1
s_common.setlogging(logger, 'WARNING')
# Ensure that SYN_DIR is available
_ = s_common.getSynDir()
async with await s_telepath.openurl(argv[0]) as item:
try:
s_version.reqVersion(item._getSynVers(), reqver)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
print(
f'Proxy version {valu} is outside of the cmdr supported range ({reqver}).'
)
print(
f'Please use a version of Synapse which supports {valu}; current version is {s_version.verstring}.'
)
return 1
await runcmdr(argv, item)
return 0
def main(argv, outp=s_output.stdout):
argp = getArgParser()
opts = argp.parse_args(argv)
s_common.setlogging(logger, opts.log_level)
dirn = s_common.gendir(opts.dmonpath)
outp.printf(f'Beginning dmon from {dirn}')
# since we're dmon main, register our certdir as global
certdir = s_common.gendir(dirn, 'certs')
# set the default certdir to the dmon certdir
s_certdir.defdir = certdir
path = os.path.join(dirn, 'dmon.yaml')
if not os.path.isfile(path):
with open(path, 'wb') as fd:
fd.write(dmonyaml.encode('utf8'))
dmon = None
try:
dmon = s_glob.sync(s_daemon.Daemon.anit(opts.dmonpath))
host, port = dmon.addr
outp.printf(f'dmon listening on: {host}:{port}')
dmon.main()
finally:
if dmon is not None:
s_glob.sync(dmon.fini())
def main(argv, outp=s_output.stdout): # pragma: no cover
teleport = os.getenv('SYN_CORTEX_PORT', '27492')
telehost = os.getenv('SYN_CORTEX_HOST', '127.0.0.1')
# httpport = os.getenv('SYN_CORTEX_HTTP_PORT', '80')
# httphost = os.getenv('SYN_CORTEX_HTTP_HOST', '127.0.0.1')
# httpsport = os.getenv('SYN_CORTEX_HTTPS_PORT', '443')
# httpshost = os.getenv('SYN_CORTEX_HTTPS_HOST', '127.0.0.1')
pars = argparse.ArgumentParser(prog='synapse.servers.cortex')
pars.add_argument('--port',
default=teleport,
help='The TCP port to bind for telepath.')
pars.add_argument('--host',
default=telehost,
help='The host address to bind telepath.')
pars.add_argument('coredir',
help='The directory for the cortex to use for storage.')
opts = pars.parse_args(argv)
s_common.setlogging(logger)
dmon = s_glob.sync(mainopts(opts, outp=outp))
return dmon.main()
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
outp.printf('starting cortex: %s' % (opts.coredir, ))
core = await s_cortex.Cortex.anit(opts.coredir)
try:
outp.printf('...cortex API (telepath): %s' % (opts.telepath, ))
await core.dmon.listen(opts.telepath)
outp.printf('...cortex API (https): %s' % (opts.port, ))
await core.addHttpsPort(opts.port)
if opts.name:
outp.printf(f'...cortex additional share name: {opts.name}')
core.dmon.share(opts.name, core)
if opts.mirror:
outp.printf(f'initializing cortex mirror of: {opts.mirror}')
await core.initCoreMirror(opts.mirror)
return core
except Exception:
await core.fini()
raise
def main(dirn, conf=None):
'''
Initialize and execute the main loop for a Cell.
Args:
dirn (str): Directory backing the Cell data.
conf (dict): Configuration dictionary.
Notes:
This ends up calling ``main()`` on the Cell, and does not return
anything. It cals sys.exit() at the end of its processing.
'''
try:
# Configure logging since we may have come in via
# multiprocessing.Process as part of a Daemon config.
s_common.setlogging(logger, os.getenv('SYN_TEST_LOG_LEVEL', 'WARNING'))
dirn = s_common.genpath(dirn)
ctor, func = getCellCtor(dirn, conf=conf)
cell = func(dirn, conf)
port = cell.getCellPort()
logger.warning('cell divided: %s (%s) port: %d' % (ctor, dirn, port))
cell.main()
sys.exit(0)
except Exception as e:
logger.exception('main: %s (%s)' % (dirn, e))
sys.exit(1)
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
outp.printf('starting axon: %s' % (opts.axondir, ))
axon = await s_axon.Axon.anit(opts.axondir)
try:
outp.printf('...axon API (telepath): %s' % (opts.telepath, ))
await axon.dmon.listen(opts.telepath)
outp.printf('...axon API (https): %s' % (opts.port, ))
await axon.addHttpsPort(opts.port)
if opts.name:
outp.printf(f'...axon additional share name: {opts.name}')
axon.dmon.share(opts.name, axon)
return axon
except Exception:
await axon.fini()
raise
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
outp.printf('starting axon: %s' % (opts.axondir,))
axon = await s_axon.Axon.anit(opts.axondir)
try:
outp.printf('...axon API (telepath): %s' % (opts.telepath,))
await axon.dmon.listen(opts.telepath)
outp.printf('...axon API (https): %s' % (opts.port,))
await axon.addHttpsPort(opts.port)
if opts.name:
outp.printf(f'...axon additional share name: {opts.name}')
axon.dmon.share(opts.name, axon)
return axon
except Exception:
await axon.fini()
raise
async def initFromArgv(cls, argv, outp=None):
'''
Cell launcher which does automatic argument parsing, environment variable resolution and Cell creation.
Args:
argv (list): A list of command line arguments to launch the Cell with.
outp (s_ouput.OutPut): Optional, an output object.
Notes:
This does the following items:
- Create a Config object from the Cell class.
- Creates an Argument Parser from the Cell class and Config object.
- Parses the provided arguments.
- Loads configuration data from the parsed options and environment variables.
- Sets logging for the process.
- Creates the Cell from the Cell Ctor.
- Adds a Telepath listener, HTTPs port listeners and Telepath share names.
- Returns the Cell.
Returns:
Cell: This returns an instance of the Cell.
'''
conf = cls.initCellConf()
pars = cls.getArgParser(conf=conf)
opts = pars.parse_args(argv)
conf.setConfFromOpts(opts)
conf.setConfFromEnvs()
s_common.setlogging(logger, defval=opts.log_level)
cell = await cls.anit(opts.dirn, conf=conf)
try:
await cell.addHttpsPort(opts.https)
await cell.dmon.listen(opts.telepath)
if opts.name is not None:
cell.dmon.share(opts.name, cell)
if outp is not None:
outp.printf(f'...{cell.getCellType()} API (telepath): %s' %
(opts.telepath, ))
outp.printf(f'...{cell.getCellType()} API (https): %s' %
(opts.https, ))
if opts.name is not None:
outp.printf(
f'...{cell.getCellType()} API (telepath name): %s' %
(opts.name, ))
except Exception:
await cell.fini()
raise
return cell
def corework(spawninfo, todo, done):
'''
Multiprocessing target for hosting a SpawnCore launched by a SpawnProc.
'''
# This logging call is okay to run since we're executing in
# our own process space and no logging has been configured.
s_common.setlogging(logger, spawninfo.get('loglevel'))
asyncio.run(_workloop(spawninfo, todo, done))
def _validateStixProc(bundle, logconf):
'''
Multiprocessing target for stix validation
'''
# This logging call is okay to run since we're executing in
# our own process space and no logging has been configured.
s_common.setlogging(logger, **logconf)
resp = validateStix(bundle)
return resp
async def main(argv): # pragma: no cover
if len(argv) != 1:
print('usage: python -m synapse.tools.cmdr <url>')
return -1
s_common.setlogging(logger, 'WARNING')
async with await s_telepath.openurl(argv[0]) as item:
cmdr = await s_cmdr.getItemCmdr(item)
await cmdr.addSignalHandlers()
await cmdr.runCmdLoop()
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
cell = await getCell(outp,
opts.celldir,
opts.cellctor,
opts.port,
opts.telepath,
name=opts.name,
)
return cell
async def main(argv, outp=s_output.stdout):
opts = parse(argv)
s_common.setlogging(logger)
cell = await getCell(
outp,
opts.celldir,
opts.cellctor,
opts.port,
opts.telepath,
name=opts.name,
)
return cell
async def main(argv, outp=None): # pragma: no cover
if outp is None:
outp = s_output.stdout
s_common.setlogging(logger, 'WARNING')
path = s_common.getSynPath('telepath.yaml')
async with contextlib.AsyncExitStack() as ctx:
telefini = await s_telepath.loadTeleEnv(path)
if telefini is not None:
ctx.push_async_callback(telefini)
await _main(argv, outp)
return 0
def main(argv, outp=None):
if outp is None:
outp = s_output.OutPut()
p = getArgParser()
opts = p.parse_args(argv)
s_common.setlogging(logger,
opts.log_level)
if opts.lsboot:
for path in lsboot():
outp.printf(path)
return
if opts.onboot:
plat = s_thishost.get('platform')
if plat not in ('linux', 'darwin'):
raise Exception('--onboot does not support platform: %s' % (plat,))
for path in opts.configs:
logger.info('onboot add: %s' % (path,))
onboot(path)
return
if opts.noboot:
for path in opts.configs:
logger.info('onboot del: %s' % (path,))
noboot(path)
return
dmon = s_daemon.Daemon()
if opts.asboot:
dmon.loadDmonFile(cfgfile)
for path in opts.configs:
dmon.loadDmonFile(path)
dmon.main()
async def main(argv): # pragma: no cover
if len(argv) not in (1, 2):
print(
'usage: python -m synapse.tools.cmdr <url> [<single quoted command>]'
)
return 1
s_common.setlogging(logger, 'WARNING')
path = s_common.getSynPath('telepath.yaml')
async with contextlib.AsyncExitStack() as ctx:
telefini = await s_telepath.loadTeleEnv(path)
if telefini is not None:
ctx.push_async_callback(telefini)
await _main(argv)
return 0
async def main(argv, outp=None): # pragma: no cover
if outp is None:
outp = s_output.stdout
if len(argv) not in (1, 2):
outp.printf('usage: python -m synapse.tools.aha.list <url> [network name]')
return 1
s_common.setlogging(logger, 'WARNING')
path = s_common.getSynPath('telepath.yaml')
async with contextlib.AsyncExitStack() as ctx:
telefini = await s_telepath.loadTeleEnv(path)
if telefini is not None:
ctx.push_async_callback(telefini)
await _main(argv, outp)
return 0
async def main(argv): # pragma: no cover
if len(argv) not in (1, 2):
print(
'usage: python -m synapse.tools.cmdr <url> [<single quoted command>]'
)
return -1
s_common.setlogging(logger, 'WARNING')
async with await s_telepath.openurl(argv[0]) as item:
cmdr = await s_cmdr.getItemCmdr(item)
await cmdr.addSignalHandlers()
# Enable colors for users
cmdr.colorsenabled = True
if len(argv) == 2:
await cmdr.runCmdLine(argv[1])
return
await cmdr.runCmdLoop()
logger = logging.getLogger(__name__)
prog = 'synapse.tools.rstorm'
descr = 'An RST pre-processor that allows you to embed storm directives.'
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser(prog=prog, description=descr)
pars.add_argument('rstfile', help='Input RST file with storm directives.')
pars.add_argument('--save', help='Output file to save (default: stdout)')
opts = pars.parse_args(argv)
async with await s_rstorm.StormRst.anit(opts.rstfile) as rstorm:
lines = await rstorm.run()
if opts.save:
with open(s_common.genpath(opts.save), 'w') as fd:
fd.truncate(0)
[fd.write(line) for line in lines]
else:
for line in lines:
outp.printf(line, addnl=False)
if __name__ == '__main__':
s_common.setlogging(logger)
sys.exit(asyncio.run(main(sys.argv[1:])))
async def _main(): # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
return await main(sys.argv[1:])
desc = 'Command line tool to generate various synapse documentation.'
pars = argparse.ArgumentParser('synapse.tools.autodoc', description=desc)
pars.add_argument('--cortex', '-c', default=None,
help='Cortex URL for model inspection')
pars.add_argument('--savedir', default=None,
help='Save output to the given directory')
doc_type = pars.add_mutually_exclusive_group()
doc_type.add_argument('--doc-model', action='store_true', default=False,
help='Generate RST docs for the DataModel within a cortex')
doc_type.add_argument('--doc-conf', default=None,
help='Generate RST docs for the Confdefs for a given Cell ctor')
pars.add_argument('--doc-conf-reflink', default=':ref:`devops-cell-config`',
help='Reference link for how to set the cell configuration options.')
pars.add_argument('--doc-conf-title', default=None, type=str,
help='Use a custom string for the document title.')
doc_type.add_argument('--doc-storm', default=None,
help='Generate RST docs for a stormssvc implemented by a given Cell')
doc_type.add_argument('--doc-stormpkg', default=None,
help='Generate RST docs for the specified Storm package YAML file.')
doc_type.add_argument('--doc-stormtypes', default=None, action='store_true',
help='Generate RST docs for StormTypes')
return pars
if __name__ == '__main__': # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
asyncio.run(main(sys.argv[1:]))
return 0
def makeargparser():
desc = 'Command line tool for ingesting data into a cortex'
pars = argparse.ArgumentParser('synapse.tools.ingest', description=desc)
muxp = pars.add_mutually_exclusive_group(required=True)
muxp.add_argument('--cortex', '-c', type=str,
help='Cortex to connect and add nodes too.')
muxp.add_argument('--test', '-t', default=False, action='store_true',
help='Perform a local ingest against a temporary cortex.')
pars.add_argument('--debug', '-d', default=False, action='store_true',
help='Drop to interactive prompt to inspect cortex after loading data.')
pars.add_argument('--format', '-f', type=str, action='store', default='syn.ingest',
help='Feed format to use for the ingested data.')
pars.add_argument('--modules', '-m', type=str, action='append', default=[],
help='Additional modules to load locally with a test Cortex.')
pars.add_argument('--chunksize', type=int, action='store', default=1000,
help='Default chunksize for iterating over items.')
pars.add_argument('--offset', type=int, action='store', default=0,
help='Item offset to start consuming msgpack files from.')
pars.add_argument('files', nargs='*', help='json/yaml/msgpack feed files')
return pars
if __name__ == '__main__': # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
asyncio.run(main(sys.argv[1:]))
def _main(argv): # pragma: no cover
s_common.setlogging(logger, defval='DEBUG')
return main(argv)
def _main(argv): # pragma: no cover
s_common.setlogging(logger, defval='DEBUG')
return main(argv)
async def _main(): # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
return await main(sys.argv[1:])
'defaultnobuid': DefaultNoBuidConf,
'dedicatedasynclogging': DedicatedAsyncLogConf,
}
'''
Benchmark cortex operations
TODO: separate client process, multiple clients
TODO: tagprops, regex, control flow, node data, multiple layers, spawn option, remote layer
'''
logger = logging.getLogger(__name__)
if __debug__:
logger.warning('Running benchmark without -O. Performance will be slower.')
s_common.setlogging(logger, 'ERROR')
async def acount(genr):
'''
Counts an async generator
'''
count = 0
async for _ in genr:
count += 1
return count
syntest = s_t_utils.SynTest()
class TestData(s_base.Base):
'''
Pregenerates a bunch of data for future test runs