def getCellCtor(dirn, conf=None): ''' Find the ctor option for a cell and resolve the function. ''' ctor = None if conf is not None: ctor = conf.get('ctor') path = s_common.genpath(dirn, 'config.json') if ctor is None and os.path.isfile(path): subconf = s_common.jsload(path) ctor = subconf.get('ctor') if ctor is None: raise s_common.ReqConfOpt(mesg='Missing ctor, cannot divide', name='ctor') func = s_dyndeps.getDynLocal(ctor) if func is None: raise s_common.NoSuchCtor(mesg='Cannot resolve ctor', name=ctor) return ctor, func
def _fireAxonIden(self, iden): axondir = s_common.gendir(self.datadir, '%s.axon' % iden) opts = self.makeAxonOpts() jsopts = s_common.jsload(axondir, 'axon.opts') if jsopts is not None: opts.update(jsopts) # Special case where the axonbus may update - we want to ensure # we're passing the latest axonbus to the Axon so it can register # itself properly. axonbus = opts.get('axon:axonbus') if axonbus is not None: myaxonbus = self.getConfOpt('axon:axonbus') if axonbus != myaxonbus: opts['axon:axonbus'] = myaxonbus s_common.jssave(opts, axondir, 'axon.opts') self.axons[iden] = Axon(axondir, **opts) bytemax = opts.get('axon:bytemax') clone = opts.get('axon:clone') if clone: self.cloneaxons.append(iden) self.usedspace = self.usedspace + bytemax
def _fireAxonIden(self, iden): axondir = s_common.gendir(self.datadir, '%s.axon' % iden) opts = dict(self.opts) jsopts = s_common.jsload(axondir, 'axon.opts') if jsopts is not None: opts.update(jsopts) self.axons[iden] = Axon(axondir, **opts)
def getItems(*paths): items = [] for path in paths: if path.endswith('.json'): item = s_common.jsload(path) if not isinstance(item, list): item = [item] items.append((path, item)) elif path.endswith(('.yaml', '.yml')): item = s_common.yamlload(path) if not isinstance(item, list): item = [item] items.append((path, item)) elif path.endswith('.mpk'): genr = s_msgpack.iterfile(path) items.append((path, genr)) else: # pragma: no cover logger.warning('Unsupported file path: [%s]', path) return items
def loadConfPath(self, path): ''' Read config options from the specified file path. Args: path (str): A file path to a json config file. Returns: None ''' full = s_common.genpath(path) if not os.path.isfile(full): return conf = s_common.jsload(path) if conf is None: return self.setConfOpts(conf)
async def _handleStormMockHttp(self, text): ''' Setup an HTTP mock file to be used with a later Storm command. Response file format: { "code": int, "body": { "data": json or a json str } } Args: text (str): Path to a json file with the response. ''' if not os.path.isfile(text): raise s_exc.NoSuchFile( mesg='Storm HTTP mock filepath does not exist', path=text) self.context['mock-http'] = s_common.jsload(text)
def getItems(*paths): items = [] for path in paths: if path.endswith('.json'): item = s_common.jsload(path) if not isinstance(item, list): item = [item] items.append((path, item)) elif path.endswith('.jsonl'): with s_common.genfile(path) as fd: item = list(s_encoding.iterdata(fd, False, format='jsonl')) items.append((path, item)) elif path.endswith(('.yaml', '.yml')): item = s_common.yamlload(path) if not isinstance(item, list): item = [item] items.append((path, item)) elif path.endswith('.mpk') or path.endswith('.nodes'): genr = s_msgpack.iterfile(path) items.append((path, genr)) else: # pragma: no cover logger.warning('Unsupported file path: [%s]', path) return items
def getCellCtor(dirn, conf=None): ''' Find the ctor option for a Cell and resolve the function. Args: dirn (str): The path to the Cell directory. This may contain the the ctor in the ``config.json`` file. conf (dict): Configuration dictionary for the cell. This may contain the ctor in the ``ctor`` key. Returns: ((str, function)): The python path to the ctor function and the resolved function. Raises: ReqConfOpt: If the ctor cannot be resolved from the cell path or conf NoSuchCtor: If the ctor function cannot be resolved. ''' ctor = None if conf is not None: ctor = conf.get('ctor') path = s_common.genpath(dirn, 'config.json') if ctor is None and os.path.isfile(path): subconf = s_common.jsload(path) ctor = subconf.get('ctor') if ctor is None: raise s_common.ReqConfOpt(mesg='Missing ctor, cannot divide', name='ctor') func = s_dyndeps.getDynLocal(ctor) if func is None: raise s_common.NoSuchCtor(mesg='Cannot resolve ctor', name=ctor) return ctor, func
def main(argv): pars = getParser() opts = pars.parse_args(argv) fps = sorted(list(glob.glob(os.path.join(opts.input, '*.json')))) raw_data = [] prefs = set() for fp in fps: fn = os.path.basename(fp) pref = get_fn_prefix(fn) prefs.add(pref) ldta = s_common.jsload(fp) ldta.setdefault('prefix', pref) raw_data.append(ldta) agg_data = aggregate_raw_data(raw_data) s_common.gendir(opts.output) crunch_data(agg_data) make_graphs(agg_data, opts.output) return 0
def getDocData(fp, root=None): ''' Args: fn (str): Name of the file to retrieve the data of. root (str): Optional root path to look for a docdata directory in. Notes: Will detect json/jsonl/yaml/mpk extensions and automatically decode that data if found; otherwise it returns bytes. Defaults to looking for the ``docdata`` directory in the current working directory. This behavior works fine for notebooks nested in the docs directory of synapse; but this root directory that is looked for may be overridden by providing an alternative root. Returns: data: May be deserialized data or bytes. Raises: ValueError if the file does not exist or directory traversal attempted.. ''' fpath = getDocPath(fp, root) if fpath.endswith('.yaml'): return s_common.yamlload(fpath) if fpath.endswith('.json'): return s_common.jsload(fpath) with s_common.genfile(fpath) as fd: if fpath.endswith('.mpk'): return s_msgpack.un(fd.read()) if fpath.endswith('.jsonl'): recs = [] for line in fd.readlines(): recs.append(json.loads(line.decode())) return recs return fd.read()
async def main(argv, outp=s_output.stdout): pars = argparse.ArgumentParser() pars.add_argument('--push', metavar='<url>', help='A telepath URL of a Cortex or PkgRepo.') pars.add_argument('--save', metavar='<path>', help='Save the completed package JSON to a file.') pars.add_argument('--optic', metavar='<path>', help='Load Optic module files from a directory.') pars.add_argument( '--no-build', action='store_true', help='Treat pkgfile argument as an already-built package') pars.add_argument( '--no-docs', default=False, action='store_true', help= 'Do not require docs to be present and replace any doc content with empty strings.' ) pars.add_argument( 'pkgfile', metavar='<pkgfile>', help= 'Path to a storm package prototype yml file, or a completed package JSON file.' ) opts = pars.parse_args(argv) if opts.no_build: pkgdef = s_common.jsload(opts.pkgfile) if opts.save: print( f'File {opts.pkgfile} is treated as already built (--no-build); incompatible with --save.', file=sys.stderr) return 1 else: pkgdef = loadPkgProto(opts.pkgfile, opticdir=opts.optic, no_docs=opts.no_docs) if not opts.save and not opts.push: print('Neither --push nor --save provided. Nothing to do.', file=sys.stderr) return 1 if opts.save: s_common.jssave(pkgdef, opts.save) if opts.push: path = s_common.genpath('~/.syn/telepath.yaml') fini = await s_telepath.loadTeleEnv(path) async with await s_telepath.openurl(opts.push) as core: await core.addStormPkg(pkgdef) if fini is not None: # pragma: no cover await fini() return 0