def getCellCtor(dirn, conf=None):
'''
Find the ctor option for a cell and resolve the function.
'''
ctor = None
if conf is not None:
ctor = conf.get('ctor')
path = s_common.genpath(dirn, 'config.json')
if ctor is None and os.path.isfile(path):
subconf = s_common.jsload(path)
ctor = subconf.get('ctor')
if ctor is None:
raise s_common.ReqConfOpt(mesg='Missing ctor, cannot divide',
name='ctor')
func = s_dyndeps.getDynLocal(ctor)
if func is None:
raise s_common.NoSuchCtor(mesg='Cannot resolve ctor',
name=ctor)
return ctor, func
def _fireAxonIden(self, iden):
axondir = s_common.gendir(self.datadir, '%s.axon' % iden)
opts = self.makeAxonOpts()
jsopts = s_common.jsload(axondir, 'axon.opts')
if jsopts is not None:
opts.update(jsopts)
# Special case where the axonbus may update - we want to ensure
# we're passing the latest axonbus to the Axon so it can register
# itself properly.
axonbus = opts.get('axon:axonbus')
if axonbus is not None:
myaxonbus = self.getConfOpt('axon:axonbus')
if axonbus != myaxonbus:
opts['axon:axonbus'] = myaxonbus
s_common.jssave(opts, axondir, 'axon.opts')
self.axons[iden] = Axon(axondir, **opts)
bytemax = opts.get('axon:bytemax')
clone = opts.get('axon:clone')
if clone:
self.cloneaxons.append(iden)
self.usedspace = self.usedspace + bytemax
def _fireAxonIden(self, iden):
axondir = s_common.gendir(self.datadir, '%s.axon' % iden)
opts = dict(self.opts)
jsopts = s_common.jsload(axondir, 'axon.opts')
if jsopts is not None:
opts.update(jsopts)
self.axons[iden] = Axon(axondir, **opts)
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
def loadConfPath(self, path):
'''
Read config options from the specified file path.
Args:
path (str): A file path to a json config file.
Returns:
None
'''
full = s_common.genpath(path)
if not os.path.isfile(full):
return
conf = s_common.jsload(path)
if conf is None:
return
self.setConfOpts(conf)
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
async def _handleStormMockHttp(self, text):
'''
Setup an HTTP mock file to be used with a later Storm command.
Response file format:
{
"code": int,
"body": {
"data": json or a json str
}
}
Args:
text (str): Path to a json file with the response.
'''
if not os.path.isfile(text):
raise s_exc.NoSuchFile(
mesg='Storm HTTP mock filepath does not exist', path=text)
self.context['mock-http'] = s_common.jsload(text)
def getItems(*paths):
items = []
for path in paths:
if path.endswith('.json'):
item = s_common.jsload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.jsonl'):
with s_common.genfile(path) as fd:
item = list(s_encoding.iterdata(fd, False, format='jsonl'))
items.append((path, item))
elif path.endswith(('.yaml', '.yml')):
item = s_common.yamlload(path)
if not isinstance(item, list):
item = [item]
items.append((path, item))
elif path.endswith('.mpk') or path.endswith('.nodes'):
genr = s_msgpack.iterfile(path)
items.append((path, genr))
else: # pragma: no cover
logger.warning('Unsupported file path: [%s]', path)
return items
def getCellCtor(dirn, conf=None):
'''
Find the ctor option for a Cell and resolve the function.
Args:
dirn (str): The path to the Cell directory. This may contain the the
ctor in the ``config.json`` file.
conf (dict): Configuration dictionary for the cell. This may contain
the ctor in the ``ctor`` key.
Returns:
((str, function)): The python path to the ctor function and the resolved function.
Raises:
ReqConfOpt: If the ctor cannot be resolved from the cell path or conf
NoSuchCtor: If the ctor function cannot be resolved.
'''
ctor = None
if conf is not None:
ctor = conf.get('ctor')
path = s_common.genpath(dirn, 'config.json')
if ctor is None and os.path.isfile(path):
subconf = s_common.jsload(path)
ctor = subconf.get('ctor')
if ctor is None:
raise s_common.ReqConfOpt(mesg='Missing ctor, cannot divide',
name='ctor')
func = s_dyndeps.getDynLocal(ctor)
if func is None:
raise s_common.NoSuchCtor(mesg='Cannot resolve ctor', name=ctor)
return ctor, func
def main(argv):
pars = getParser()
opts = pars.parse_args(argv)
fps = sorted(list(glob.glob(os.path.join(opts.input, '*.json'))))
raw_data = []
prefs = set()
for fp in fps:
fn = os.path.basename(fp)
pref = get_fn_prefix(fn)
prefs.add(pref)
ldta = s_common.jsload(fp)
ldta.setdefault('prefix', pref)
raw_data.append(ldta)
agg_data = aggregate_raw_data(raw_data)
s_common.gendir(opts.output)
crunch_data(agg_data)
make_graphs(agg_data, opts.output)
return 0
def getDocData(fp, root=None):
'''
Args:
fn (str): Name of the file to retrieve the data of.
root (str): Optional root path to look for a docdata directory in.
Notes:
Will detect json/jsonl/yaml/mpk extensions and automatically
decode that data if found; otherwise it returns bytes.
Defaults to looking for the ``docdata`` directory in the current
working directory. This behavior works fine for notebooks nested
in the docs directory of synapse; but this root directory that
is looked for may be overridden by providing an alternative root.
Returns:
data: May be deserialized data or bytes.
Raises:
ValueError if the file does not exist or directory traversal attempted..
'''
fpath = getDocPath(fp, root)
if fpath.endswith('.yaml'):
return s_common.yamlload(fpath)
if fpath.endswith('.json'):
return s_common.jsload(fpath)
with s_common.genfile(fpath) as fd:
if fpath.endswith('.mpk'):
return s_msgpack.un(fd.read())
if fpath.endswith('.jsonl'):
recs = []
for line in fd.readlines():
recs.append(json.loads(line.decode()))
return recs
return fd.read()
def getDocData(fp, root=None):
'''
Args:
fn (str): Name of the file to retrieve the data of.
root (str): Optional root path to look for a docdata directory in.
Notes:
Will detect json/jsonl/yaml/mpk extensions and automatically
decode that data if found; otherwise it returns bytes.
Defaults to looking for the ``docdata`` directory in the current
working directory. This behavior works fine for notebooks nested
in the docs directory of synapse; but this root directory that
is looked for may be overridden by providing an alternative root.
Returns:
data: May be deserialized data or bytes.
Raises:
ValueError if the file does not exist or directory traversal attempted..
'''
fpath = getDocPath(fp, root)
if fpath.endswith('.yaml'):
return s_common.yamlload(fpath)
if fpath.endswith('.json'):
return s_common.jsload(fpath)
with s_common.genfile(fpath) as fd:
if fpath.endswith('.mpk'):
return s_msgpack.un(fd.read())
if fpath.endswith('.jsonl'):
recs = []
for line in fd.readlines():
recs.append(json.loads(line.decode()))
return recs
return fd.read()
async def main(argv, outp=s_output.stdout):
pars = argparse.ArgumentParser()
pars.add_argument('--push',
metavar='<url>',
help='A telepath URL of a Cortex or PkgRepo.')
pars.add_argument('--save',
metavar='<path>',
help='Save the completed package JSON to a file.')
pars.add_argument('--optic',
metavar='<path>',
help='Load Optic module files from a directory.')
pars.add_argument(
'--no-build',
action='store_true',
help='Treat pkgfile argument as an already-built package')
pars.add_argument(
'--no-docs',
default=False,
action='store_true',
help=
'Do not require docs to be present and replace any doc content with empty strings.'
)
pars.add_argument(
'pkgfile',
metavar='<pkgfile>',
help=
'Path to a storm package prototype yml file, or a completed package JSON file.'
)
opts = pars.parse_args(argv)
if opts.no_build:
pkgdef = s_common.jsload(opts.pkgfile)
if opts.save:
print(
f'File {opts.pkgfile} is treated as already built (--no-build); incompatible with --save.',
file=sys.stderr)
return 1
else:
pkgdef = loadPkgProto(opts.pkgfile,
opticdir=opts.optic,
no_docs=opts.no_docs)
if not opts.save and not opts.push:
print('Neither --push nor --save provided. Nothing to do.',
file=sys.stderr)
return 1
if opts.save:
s_common.jssave(pkgdef, opts.save)
if opts.push:
path = s_common.genpath('~/.syn/telepath.yaml')
fini = await s_telepath.loadTeleEnv(path)
async with await s_telepath.openurl(opts.push) as core:
await core.addStormPkg(pkgdef)
if fini is not None: # pragma: no cover
await fini()
return 0
