def main(argv, outp=None): if outp is None: outp = s_output.OutPut() p = getArgParser() opts = p.parse_args(argv) core = s_telepath.openurl(opts.cortex) tags = [] if opts.tags: for tag in opts.tags.split(','): tags.append(tag) if tags: outp.printf('adding tags: %r' % (tags, )) for path in opts.filenames: with open(path, 'rb') as fd: base = os.path.basename(path) node = core.formNodeByFd(fd, name=base) core.addTufoTags(node, tags) iden = node[1].get('file:bytes') size = node[1].get('file:bytes:size') name = node[1].get('file:bytes:name') outp.printf('file: %s (%d) added (%s) as %s' % (base, size, iden, name))
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = getArgParser() opts = pars.parse_args(argv) for path in opts.paths: if not path.endswith('.json'): outp.printf('skip: %s (not .json extension)' % (path, )) continue if not os.path.isfile(path): outp.printf('skip: %s (not a file)' % (path, )) continue base = path[:-5] newp = base + '.mpk' outp.printf('converting: %s -> .mpk' % (path, )) with open(path, 'r', encoding='utf8') as fd: with open(newp, 'wb') as pk: for line in fd: item = json.loads(line) pk.write(s_msgpack.en(item)) if opts.rm: os.unlink(path)
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() parser = makeargpaser() opts = parser.parse_args(argv) if not opts.verbose: logging.disable(logging.DEBUG) if os.path.isfile(opts.output) and not opts.force: outp.printf('Cannot overwrite a backup.') return 1 genrows_kwargs = {} if opts.extra_args: with open(opts.extra_args, 'rb') as fd: genrows_kwargs = json.loads(fd.read().decode()) storconf = {'rev:storage': False} if opts.revstorage: storconf['rev:storage'] = True backup_tufo = gen_backup_tufo(opts) with open(opts.output, 'wb') as fd: fd.write(s_msgpack.en(backup_tufo)) with s_cortex.openstore(opts.store, storconf=storconf) as store: dump_store(outp, fd, store, compress=opts.compress, dump_blobstore=opts.dump_blobstore, genrows_kwargs=genrows_kwargs) outp.printf('Fin') return 0
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument('--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument('--configable-opts', action='store_true', default=False, help='Generate RST docs of the Configable classes in Synapse.') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) fd = None if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) if opts.doc_model: with s_cortex.openurl(opts.cortex) as core: return docModel(outp, fd, core) if opts.configable_opts: return docConfigables(outp, fd)
async def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) if opts.doc_model: if opts.cortex: async with await s_telepath.openurl(opts.cortex) as core: rsttypes, rstforms = await docModel(outp, core) else: async with s_cortex.getTempCortex() as core: rsttypes, rstforms = await docModel(outp, core) if opts.savedir: with open(s_common.genpath(opts.savedir, 'datamodel_types.rst'), 'wb') as fd: fd.write(rsttypes.getRstText().encode()) with open(s_common.genpath(opts.savedir, 'datamodel_forms.rst'), 'wb') as fd: fd.write(rstforms.getRstText().encode()) return 0
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() parser = makeargpaser() opts = parser.parse_args(argv) if not opts.verbose: logging.disable(logging.DEBUG) # Check to see if we're working with a savefile or a dumprows file decompress = False discard_first_event = False with open(opts.input, 'rb') as fd: gen = s_msgpack.iterfd(fd) tufo0 = next(gen) if tufo0[0] == 'syn:cortex:rowdump:info': outp.printf('Restoring from a dumprows file.') discard_first_event = True decompress = tufo0[1].get('rows:compress') if decompress: outp.printf('Gzip row compression enabled.') else: outp.printf('Restoring from a savefile') # No longer need that generator around with the dangler to fd del gen storconf = {'rev:storage': False} if opts.revstorage: # pragma: no cover storconf['rev:storage'] = True with open(opts.input, 'rb') as fd: gen = s_msgpack.iterfd(fd) if discard_first_event: next(gen) with s_cortex.openstore(opts.store, storconf=storconf) as store: outp.printf('Starting row level restore') tick = time.time() i = 0 nrows = 0 for event in gen: if decompress and 'rows' in event[1]: event[1]['rows'] = s_msgpack.un( gzip.decompress(event[1].get('rows'))) i += 1 if i % 250 == 0: outp.printf('Loaded {} events'.format(i)) store.loadbus.dist(event) _nrows = len(event[1].get('rows', ())) nrows += _nrows if _nrows and i % 10 == 0: logger.debug('Loaded %s rows', nrows) tock = time.time() outp.printf('Done loading events - took {} seconds.'.format(tock - tick)) outp.printf('Fin') return 0
async def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) if opts.doc_model: if opts.cortex: async with await s_telepath.openurl(opts.cortex) as core: rsttypes, rstforms = await docModel(outp, core) else: async with s_cortex.getTempCortex() as core: rsttypes, rstforms = await docModel(outp, core) if opts.savedir: with open(s_common.genpath(opts.savedir, 'datamodel_types.rst'), 'wb') as fd: fd.write(rsttypes.getRstText().encode()) with open(s_common.genpath(opts.savedir, 'datamodel_forms.rst'), 'wb') as fd: fd.write(rstforms.getRstText().encode()) if opts.doc_conf: confdocs, cname = await docConfdefs(opts.doc_conf, reflink=opts.doc_conf_reflink, doc_title=opts.doc_conf_title, ) if opts.savedir: with open(s_common.genpath(opts.savedir, f'conf_{cname.lower()}.rst'), 'wb') as fd: fd.write(confdocs.getRstText().encode()) if opts.doc_storm: confdocs, svcname = await docStormsvc(opts.doc_storm) if opts.savedir: with open(s_common.genpath(opts.savedir, f'stormsvc_{svcname.lower()}.rst'), 'wb') as fd: fd.write(confdocs.getRstText().encode()) if opts.doc_stormpkg: pkgdocs, pkgname = await docStormpkg(opts.doc_stormpkg) if opts.savedir: with open(s_common.genpath(opts.savedir, f'stormpkg_{pkgname.lower()}.rst'), 'wb') as fd: fd.write(pkgdocs.getRstText().encode()) if opts.doc_stormtypes: libdocs, typedocs = await docStormTypes() if opts.savedir: with open(s_common.genpath(opts.savedir, f'stormtypes_libs.rst'), 'wb') as fd: fd.write(libdocs.getRstText().encode()) with open(s_common.genpath(opts.savedir, f'stormtypes_prims.rst'), 'wb') as fd: fd.write(typedocs.getRstText().encode()) return 0
async def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) if opts.offset: if len(opts.files) > 1: outp.printf( 'Cannot start from a arbitrary offset for more than 1 file.') return 1 outp.printf( f'Starting from offset [{opts.offset}] - it may take a while' f' to get to that location in the input file.') if opts.test: async with s_cortex.getTempCortex(mods=opts.modules) as prox: await addFeedData(prox, outp, opts.format, opts.debug, chunksize=opts.chunksize, offset=opts.offset, *opts.files) elif opts.cortex: async with await s_telepath.openurl(opts.cortex) as core: try: s_version.reqVersion(core._getSynVers(), reqver) except s_exc.BadVersion as e: valu = s_version.fmtVersion(*e.get('valu')) outp.printf( f'Cortex version {valu} is outside of the feed tool supported range ({reqver}).' ) outp.printf( f'Please use a version of Synapse which supports {valu}; ' f'current version is {s_version.verstring}.') return 1 await addFeedData(core, outp, opts.format, opts.debug, chunksize=opts.chunksize, offset=opts.offset, *opts.files) else: # pragma: no cover outp.printf('No valid options provided [%s]', opts) return 1 return 0
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() p = getArgParser() opts = p.parse_args(argv) log_level = os.getenv('SYN_DMON_LOG_LEVEL', opts.log_level) if log_level: # pragma: no cover log_level = log_level.upper() if log_level not in LOG_LEVEL_CHOICES: raise ValueError( 'Invalid log level provided: {}'.format(log_level)) logging.basicConfig( level=log_level, format= '%(asctime)s [%(levelname)s] %(message)s [%(filename)s:%(funcName)s]' ) logger.info('log level set to ' + log_level) if opts.lsboot: for path in lsboot(): outp.printf(path) return if opts.onboot: plat = s_thishost.get('platform') if plat not in ('linux', 'darwin'): raise Exception('--onboot does not support platform: %s' % (plat, )) for path in opts.configs: logger.info('onboot add: %s' % (path, )) onboot(path) return if opts.noboot: for path in opts.configs: logger.info('onboot del: %s' % (path, )) noboot(path) return dmon = s_daemon.Daemon() if opts.asboot: dmon.loadDmonFile(cfgfile) for path in opts.configs: dmon.loadDmonFile(path) dmon.main()
def __init__(self, outp=None): EventBus.__init__(self) if outp == None: outp = s_output.OutPut() self.outp = outp self.cmds = {} self.cmdprompt = 'cli> ' self.addCmdClass( CmdHelp ) self.addCmdClass( CmdQuit )
async def main(argv, outprint=None): if outprint is None: # pragma: no cover outprint = s_output.OutPut() global outp outp = outprint pars = makeargparser() try: opts = pars.parse_args(argv) except s_exc.ParserExit: return -1 return await opts.func(opts)
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargpaser() opts = pars.parse_args(argv) if opts.module: mod = s_dyndeps.tryDynMod(opts.module) outp.printf(f'Loaded {opts.module}@{mod}') if opts.cells: outp.printf('Registered cells:') for cname, cpath in s_cells.getCells(): outp.printf(f'{cname:<10} {cpath:>10}') return 0 dirn = s_common.genpath(opts.dmonpath, 'cells', opts.cellname) if os.path.isdir(dirn): outp.printf(f'cell directory already exists: {dirn}') return 1 dmon = {} if opts.listen: dmon['listen'] = opts.listen if opts.module: dmon['modules'] = [opts.module] if dmon: dmon.setdefault('modules', []) dmon_fp = os.path.join(opts.dmonpath, 'dmon.yaml') if os.path.exists(dmon_fp): outp.printf(f'Cannot overwrite existing dmon.yaml file. [{dmon_fp}]') return 1 s_common.yamlsave(dmon, dmon_fp) boot = { 'cell:name': opts.cellname, } if opts.auth: boot['auth:en'] = True if opts.admin: boot['auth:en'] = True boot['auth:admin'] = opts.admin outp.printf(f'Deploying a {opts.celltype} at: {dirn}') s_cells.deploy(opts.celltype, dirn, boot) return 0
def __init__(self, item, outp=None, **locs): EventBus.__init__(self) if outp is None: outp = s_output.OutPut() self.outp = outp self.locs = locs self.item = item # whatever object we are commanding self.cmds = {} self.cmdprompt = 'cli> ' self.addCmdClass(CmdHelp) self.addCmdClass(CmdQuit)
async def main(argv, outp=None): pars = setup() opts = pars.parse_args(argv) path = s_common.getSynPath('telepath.yaml') telefini = await s_telepath.loadTeleEnv(path) if outp is None: # pragma: no cover outp = s_output.OutPut() if opts.output is None: opts.output = '.' outdir = pathlib.Path(opts.output) s_common.gendir(opts.output) async with await s_telepath.openurl(opts.axon) as axon: # reminder: these are the hashes *not* available awants = await axon.wants([s_common.uhex(h) for h in opts.hashes]) for a in awants: outp.printf(f'{s_common.ehex(a)} not in axon store') exists = [h for h in opts.hashes if s_common.uhex(h) not in awants] for h in exists: try: outp.printf(f'Fetching {h} to file') with open(outdir.joinpath(h), 'wb') as fd: async for b in axon.get(s_common.uhex(h)): fd.write(b) outp.printf(f'Fetched {h} to file') except Exception as e: outp.printf('Error: Hit Exception: %s' % (str(e), )) continue if telefini: # pragma: no cover await telefini() return 0
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() p = getArgParser() opts = p.parse_args(argv) if opts.log_level: logging.basicConfig(level=opts.log_level.upper()) logger.info('log level set to ' + opts.log_level) if opts.lsboot: for path in lsboot(): outp.printf(path) return if opts.onboot: plat = s_thishost.get('platform') if plat not in ('linux', 'darwin'): raise Exception('--onboot does not support platform: %s' % (plat, )) for path in opts.configs: logger.info('onboot add: %s' % (path, )) onboot(path) return if opts.noboot: for path in opts.configs: logger.info('onboot del: %s' % (path, )) noboot(path) return dmon = s_daemon.Daemon() if opts.asboot: dmon.loadDmonFile(cfgfile) for path in opts.configs: dmon.loadDmonFile(path) dmon.main()
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) if opts.offset: if len(opts.files) > 1: outp.printf( 'Cannot start from a arbitrary offset for more than 1 file.') return 1 outp.printf( f'Starting from offset [{opts.offset}] - it may take a while' f' to get to that location in the input file.') if opts.test: with s_cortex.getTempCortex(opts.modules) as prox: addFeedData(prox, outp, opts.format, opts.debug, chunksize=opts.chunksize, offset=opts.offset, *opts.files) elif opts.cortex: with s_telepath.openurl(opts.cortex) as core: addFeedData(core, outp, opts.format, opts.debug, chunksize=opts.chunksize, offset=opts.offset, *opts.files) else: # pragma: no cover outp.printf('No valid options provided [%s]', opts) return 1 return 0
def main(argv, outp=None): pars = setup() opts = pars.parse_args(argv) if outp is None: # pragma: no cover outp = s_output.OutPut() if opts.output is None: opts.output = '.' outdir = pathlib.Path(opts.output) s_common.gendir(opts.output) with s_telepath.openurl(opts.axon) as axon: # reminder: these are the hashes *not* available awants = axon.wants([binascii.unhexlify(h) for h in opts.hashes]) for a in awants: outp.printf(f'{binascii.hexlify(a)} not in axon store') exists = [ h for h in opts.hashes if binascii.unhexlify(h) not in awants ] for h in exists: try: outp.printf(f'Fetching {h} to file') with open(outdir.joinpath(h), 'wb') as fd: for b in axon.get(binascii.unhexlify(h)): fd.write(b) outp.printf(f'Fetched {h} to file') except Exception as e: outp.printf('Error: Hit Exception: %s' % (str(e), )) continue return 0
async def __anit__(self, item, outp=None, **locs): await s_base.Base.__anit__(self) # Tell prompt_toolkit to use the asyncio event loop. use_asyncio_event_loop() if outp is None: outp = s_output.OutPut() self.outp = outp self.locs = locs self.cmdtask = None # type: asyncio.Task self.sess = None self.vi_mode = _inputrc_enables_vi_mode() self.item = item # whatever object we are commanding self.echoline = False self.colorsenabled = False if isinstance(self.item, s_base.Base): self.item.onfini(self._onItemFini) self.locs['syn:local:version'] = s_version.verstring if isinstance(self.item, s_telepath.Proxy): version = self.item._getSynVers() if version is None: # pragma: no cover self.locs[ 'syn:remote:version'] = 'Remote Synapse version unavailable' else: self.locs['syn:remote:version'] = '.'.join( [str(v) for v in version]) self.cmds = {} self.cmdprompt = 'cli> ' self.addCmdClass(CmdHelp) self.addCmdClass(CmdQuit) self.addCmdClass(CmdLocals)
async def main(argv, outprint=None): if outprint is None: # pragma: no cover outprint = s_output.OutPut() global outp outp = outprint async with contextlib.AsyncExitStack() as cm: teleyaml = s_common.getSynPath('telepath.yaml') if os.path.isfile(teleyaml): fini = await s_telepath.loadTeleEnv(teleyaml) cm.push_async_callback(fini) pars = makeargparser() try: opts = pars.parse_args(argv) except s_exc.ParserExit: return -1 retn = await opts.func(opts) return retn
def __init__(self, item, outp=None, **locs): s_eventbus.EventBus.__init__(self) if outp is None: outp = s_output.OutPut() self.outp = outp self.locs = locs self.item = item # whatever object we are commanding self.echoline = False self.finikill = False self.loopthread = None if isinstance(item, (s_base.Base, s_eventbus.EventBus)): self.item.onfini(self._onItemFini) self.cmds = {} self.cmdprompt = 'cli> ' self.addCmdClass(CmdHelp) self.addCmdClass(CmdQuit) self.addCmdClass(CmdLocals)
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() p = makeargpaser() opts = p.parse_args(args=argv) results = [] for fp in opts.input: try: guid, hashd = compute_hashes(fp=fp) except: outp.printf('Failed to compute superhash for {}'.format(fp)) else: results.append((fp, guid, hashd)) if opts.ingest: ret = [] for fp, guid, hashd in results: hashd['name'] = os.path.basename(fp) d = {"props": hashd} hl = [guid, d] ret.append(hl) if len(ret) == 1: ret = ret[0] outp.printf(json.dumps(ret, sort_keys=True, indent=2)) else: for fp, guid, hashd in results: outp.printf('Superhash for: {}'.format(fp)) hashd['guid'] = guid keys = list(hashd.keys()) keys.sort() for key in keys: value = hashd.get(key) outp.printf('{}\t{}'.format(key, value)) return 0
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = makeargparser() opts = pars.parse_args(argv) axon = s_telepath.openurl(opts.axon) core = None if opts.cortex: core = s_telepath.openurl(opts.cortex) tags = {} if opts.tags: for tag in opts.tags.split(','): tags[tag] = (None, None) if tags: outp.printf('adding tags: %r' % (list(tags.keys()))) filepaths = set() for item in opts.filenames: paths = glob.glob(item, recursive=opts.recursive) if not paths: outp.printf(f'filepath does not contain any files: {item}') continue filepaths.update([path for path in paths if os.path.isfile(path)]) for path in filepaths: bname = os.path.basename(path) hset = s_hashset.HashSet() with s_common.reqfile(path) as fd: hset.eatfd(fd) fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes} sha256 = fhashes.get('sha256') bsha256 = s_common.uhex(sha256) if not axon.has(bsha256): with axon.upload() as upfd: with s_common.genfile(path) as fd: for byts in s_common.iterfd(fd): upfd.write(byts) size, hashval = upfd.save() if hashval != bsha256: # pragma: no cover raise s_exc.SynErr(mesg='hashes do not match', ehash=s_common.ehex(hashval), ahash=hashval) outp.printf(f'Uploaded [{bname}] to axon') else: outp.printf(f'Axon already had [{bname}]') if core: pnode = (('file:bytes', f'sha256:{sha256}'), { 'props': { 'md5': fhashes.get('md5'), 'sha1': fhashes.get('sha1'), 'sha256': fhashes.get('sha256'), 'size': hset.size, 'name': bname, }, 'tags': tags, }) node = list(core.addNodes([pnode]))[0] iden = node[0][1] size = node[1]['props']['size'] name = node[1]['props']['name'] mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}' outp.printf(mesg) s_glob.sync(axon.fini()) if core: s_glob.sync(core.fini()) return 0
async def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() path = s_common.getSynPath('telepath.yaml') telefini = await s_telepath.loadTeleEnv(path) pars = makeargparser() opts = pars.parse_args(argv) axon = await s_telepath.openurl(opts.axon) core = None if opts.cortex: core = await s_telepath.openurl(opts.cortex) tags = set() if opts.tags: for tag in opts.tags.split(','): tags.add(tag) tags = tuple(tags) if tags: outp.printf(f'adding tags: {tags}') filepaths = set() for item in opts.filenames: paths = glob.glob(item, recursive=opts.recursive) if not paths: outp.printf(f'filepath does not contain any files: {item}') continue filepaths.update([path for path in paths if os.path.isfile(path)]) for path in filepaths: bname = os.path.basename(path) hset = s_hashset.HashSet() with s_common.reqfile(path) as fd: hset.eatfd(fd) fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes} sha256 = fhashes.get('sha256') bsha256 = s_common.uhex(sha256) if not await axon.has(bsha256): async with await axon.upload() as upfd: with s_common.genfile(path) as fd: for byts in s_common.iterfd(fd): await upfd.write(byts) size, hashval = await upfd.save() if hashval != bsha256: # pragma: no cover raise s_exc.SynErr(mesg='hashes do not match', ehash=s_common.ehex(hashval), ahash=hashval) outp.printf(f'Uploaded [{bname}] to axon') else: outp.printf(f'Axon already had [{bname}]') if core: opts = { 'vars': { 'md5': fhashes.get('md5'), 'sha1': fhashes.get('sha1'), 'sha256': fhashes.get('sha256'), 'size': hset.size, 'name': bname, 'tags': tags, } } q = '[file:bytes=$sha256 :md5=$md5 :sha1=$sha1 :size=$size :name=$name] ' \ '{ for $tag in $tags { [+#$tag] } }' msgs = await core.storm(q, opts=opts).list() node = [m[1] for m in msgs if m[0] == 'node'][0] iden = node[0][1] size = node[1]['props']['size'] name = node[1]['props']['name'] mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}' outp.printf(mesg) await axon.fini() if core: await core.fini() if telefini: # pragma: no cover await telefini() return 0
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='easycert', description=descr) pars.add_argument('--certdir', default='~/.syn/certs', help='Directory for certs/keys') pars.add_argument('--signas', help='sign the new cert with the given cert name') pars.add_argument('--ca', default=False, action='store_true', help='mark the certificate as a CA/CRL signer') pars.add_argument('--server', default=False, action='store_true', help='mark the certificate as a server') pars.add_argument('--server-sans', help='server cert subject alternate names') pars.add_argument('--csr', default=False, action='store_true', help='generate a cert signing request') pars.add_argument('--sign-csr', default=False, action='store_true', help='sign a cert signing request') pars.add_argument( 'name', help='common name for the certificate (or filename for CSR signing)') opts = pars.parse_args(argv) cdir = s_certdir.CertDir(path=opts.certdir) try: if opts.sign_csr: if opts.signas is None: outp.printf('--sign-csr requires --signas') return -1 xcsr = cdir._loadCsrPath(opts.name) if xcsr is None: outp.printf('csr not found: %s' % (opts.name, )) return -1 if opts.server: cdir.signHostCsr(xcsr, opts.signas, outp=outp) return 0 cdir.signUserCsr(xcsr, opts.signas, outp=outp) return 0 if opts.csr: if opts.ca: cdir.genCaCsr(opts.name, outp=outp) raise Exception('CSR for CA cert not supported (yet)') if opts.server: cdir.genHostCsr(opts.name, outp=outp) return 0 cdir.genUserCsr(opts.name, outp=outp) return 0 if opts.ca: cdir.genCaCert(opts.name, signas=opts.signas, outp=outp) return 0 if opts.server: cdir.genHostCert(opts.name, signas=opts.signas, outp=outp, sans=opts.server_sans) return 0 cdir.genUserCert(opts.name, signas=opts.signas, outp=outp) return 0 except s_common.DupFileName as e: outp.printf('file exists: %s' % (e.errinfo.get('path'), )) return -1
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() outp.printf(s_common.guid())
def test_output(self): outp = s_output.OutPut() outp.printf('foo') outp.printf('bar')
def main(argv, outp=None): if outp is None: # pragma: no cover outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='ingest', description='Command line tool for ingesting data into a cortex') pars.add_argument('--core', default='ram://', help='Cortex to use for ingest deconfliction') pars.add_argument('--progress', default=False, action='store_true', help='Print loading progress') pars.add_argument('--sync', default=None, help='Sync to an additional cortex') pars.add_argument('--save', default=None, help='Save cortex sync events to a file') pars.add_argument('--debug', default=False, action='store_true', help='Drop to interactive prompt to inspect cortex') pars.add_argument('--verbose', default=False, action='store_true', help='Show changes to local cortex incrementally') pars.add_argument('files', nargs='*', help='JSON ingest definition files') opts = pars.parse_args(argv) core = s_cortex.openurl(opts.core) try: s_telepath.reqNotProxy(core) except s_common.MustBeLocal: outp.printf('Ingest requires a local cortex to deconflict against, not a Telepath proxy') raise core.setConfOpt('enforce', 1) if opts.debug: # pragma: no cover core.setConfOpt('log:save', 1) # FIXME check for telepath proxy and bitch. # this core may not be remote because we use # the transaction API. def _print_tufo_add(mesg): tufo = mesg[1].get('node') form = tufo[1].get('tufo:form') outp.printf('add: %s=%s' % (form, tufo[1].get(form))) for prop, valu in sorted(s_tufo.props(tufo).items()): outp.printf(' :%s = %s' % (prop, valu)) def _print_tufo_tag_add(mesg): tag = mesg[1].get('tag') tufo = mesg[1].get('node') form = tufo[1].get('tufo:form') outp.printf('tag: %s=%s (%s)' % (form, tufo[1].get(form), tag)) progtot = collections.defaultdict(int) proglast = collections.defaultdict(int) proglocs = {'tick': None, 'datatot': 0, 'datalast': 0, 'tufotot': 0, 'tufolast': 0} def onGestData(mesg): proglocs['datatot'] += 1 proglocs['datalast'] += 1 def onNodeAdd(mesg): proglocs['tufotot'] += 1 proglocs['tufolast'] += 1 # callback for displaying progress... def onGestProg(mesg): act = mesg[1].get('act') progtot[act] += 1 proglast[act] += 1 progtot['total'] += 1 proglast['total'] += 1 progtick = proglocs.get('tick') if progtick is None: proglocs['tick'] = time.time() return if progtick is not None: now = time.time() delta = now - progtick if delta >= 1.0: tot = sum(proglast.values()) persec = int(float(tot) / delta) tot = proglast.get('total', 0) datatot = proglocs.get('datatot', 0) datalast = proglocs.get('datalast', 0) datasec = int(float(datalast) / delta) tufotot = proglocs.get('tufotot', 0) tufolast = proglocs.get('tufolast', 0) tufosec = int(float(tufolast) / delta) totstat = tuple(sorted(progtot.items())) laststat = tuple(sorted(proglast.items())) totstr = ' '.join(['%s=%s' % (n, v) for (n, v) in totstat]) laststr = ' '.join(['%s=%s' % (n, v) for (n, v) in laststat]) outp.printf('data: %s %s/sec (%d) nodes: %s %s/sec (%d)' % (datalast, datasec, datatot, tufolast, tufosec, tufotot)) proglast.clear() proglocs['tick'] = time.time() proglocs['datalast'] = 0 proglocs['tufolast'] = 0 if opts.save: outp.printf('saving sync events to: %s' % (opts.save,)) core.addSpliceFd(s_common.genfile(opts.save)) if opts.verbose: core.on('node:add', _print_tufo_add) core.on('node:tag:add', _print_tufo_tag_add) pump = None if opts.sync is not None: sync = s_cortex.openurl(opts.sync) pump = core.getSplicePump(sync) tick = time.time() with core.getCoreXact() as xact: for path in opts.files: gest = s_ingest.loadfile(path) if opts.progress: core.on('node:add', onNodeAdd) gest.on('gest:data', onGestData) gest.on('gest:prog', onGestProg) gest.ingest(core) tock = time.time() outp.printf('ingest took: %s sec' % (tock - tick,)) if opts.debug: # pragma: no cover s_cmdr.runItemCmdr(core) if pump is not None: pump.done() outp.printf('waiting on sync pump...') pump.waitfini() return 0
def main(argv, outp=None): if outp is None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument( '--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument( '--configable-opts', action='store_true', default=False, help='Generate RST docs of the Configable classes in Synapse.') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) fd = None if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) core = s_cortex.openurl(opts.cortex) if opts.doc_model: forms = [] types = [] props = collections.defaultdict(list) for tufo in core.getTufosByProp('syn:type'): name = tufo[1].get('syn:type') info = s_tufo.props(tufo) types.append((name, info)) for tufo in core.getTufosByProp('syn:form'): name = tufo[1].get('syn:form') info = s_tufo.props(tufo) forms.append((name, info)) for tufo in core.getTufosByProp('syn:prop'): prop = tufo[1].get('syn:prop') form = tufo[1].get('syn:prop:form') info = s_tufo.props(tufo) props[form].append((prop, info)) types.sort() forms.sort() [v.sort() for v in props.values()] rst = RstHelp() rst.addHead('Synapse Data Model', lvl=0) rst.addHead('Types', lvl=1) for name, info in types: rst.addHead(name, lvl=2) inst = core.getTypeInst(name) ex = inst.get('ex') doc = inst.get('doc') if doc is not None: rst.addLines(doc) bases = core.getTypeBases(name) rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '') if ex is not None: #valu = core.getTypeParse(name,ex) #vrep = reprvalu(valu) rst.addLines('', 'Examples:', '') rst.addLines('- repr mode: %s' % (repr(ex), )) #rst.addLines('- system mode: %s' % (vrep,)) rst.addLines('') cons = [] xforms = [] if core.isSubType(name, 'str'): regex = inst.get('regex') if regex is not None: cons.append('- regex: %s' % (regex, )) lower = inst.get('lower') if lower: xforms.append('- case: lower') restrip = inst.get('restrip') if restrip is not None: xforms.append('- regex strip: %s' % (restrip, )) nullval = inst.get('nullval') if nullval is not None: cons.append('- null value: %s' % (nullval, )) if core.isSubType(name, 'int'): minval = inst.get('min') if minval is not None: cons.append('- min value: %d (0x%x)' % (minval, minval)) maxval = inst.get('max') if maxval is not None: cons.append('- max value: %d (0x%x)' % (maxval, maxval)) ismin = inst.get('ismin') if ismin is not None: xforms.append('- is minimum: True') ismax = inst.get('ismax') if ismax is not None: xforms.append('- is maximum: True') if core.isSubType(name, 'sepr'): sep = inst.get('sep') fields = inst.get('fields') parts = [] for part in fields.split('|'): name, stype = part.split(',') parts.append(stype) seprs = sep.join(['<%s>' % p for p in parts]) rst.addLines('', 'Sepr Fields: %s' % (seprs, )) if cons: cons.append('') rst.addLines('', 'Type Constraints:', '', *cons) if xforms: xforms.append('') rst.addLines('', 'Type Transforms:', '', *xforms) rst.addHead('Forms', lvl=1) for name, info in forms: ftype = info.get('ptype', 'str') rst.addHead('%s = <%s>' % (name, ftype), lvl=2) doc = core.getPropInfo(name, 'doc') if doc is not None: rst.addLines(doc) rst.addLines('', 'Properties:', '') for prop, pnfo in props.get(name, ()): # use the resolver funcs that will recurse upward pex = core.getPropInfo(prop, 'ex') pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) defval = pnfo.get('defval') if defval is not None: pline += ' (default: %r)' % (defval, ) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc, )) outp.printf(rst.getRstText()) return 0 if opts.configable_opts: rst = RstHelp() rst.addHead('Synapse Configable Classes', lvl=0) rst.addLines( 'The following objects are Configable objects. They have' ' settings which may be provided at runtime or during object' ' initialization which may change their behavior.') basename = 'synapse' confdetails = collections.defaultdict(list) for root, dirs, files in os.walk(base_synaspe_dir): if any([v for v in dir_skips if v in root]): continue for fn in files: if fn in fn_skips: continue if not fn.endswith('.py'): continue modname = fn.rsplit('.', 1)[0] _modpath = root[len(base_synaspe_dir) + 1:].replace( os.sep, '.') modpath = '.'.join( [v for v in [basename, _modpath, modname] if v]) mod = importlib.import_module(modpath) for modattr, valu, name, results in inspect_mod( mod, cls=s_config.Configable): confdetails[modpath].append((modattr, name, results)) # Collapse details into a modpath -> Details struct detaildict = collections.defaultdict(list) for modpath, details in confdetails.items(): for detail in details: modattr, name, results = detail obj_path = '.'.join([modpath, modattr]) if obj_path in obj_path_skips: continue for rslt in results: if not rslt: continue detaildict[obj_path].append(rslt) # Now make the RST proper like keys = list(detaildict.keys()) keys.sort() for obj_path in keys: details = detaildict.get(obj_path, []) details.sort(key=lambda x: x[0]) rst.addHead(name=obj_path, lvl=1) for detail in details: confvalu, confdict = detail[0], detail[1] rst.addHead(confvalu, lvl=2) _keys = list(confdict.keys()) _keys.sort() for _key in _keys: v = confdict.get(_key) line = ' - {}: {}'.format(_key, v) rst.addLines(line) outp.printf(rst.getRstText()) if fd: fd.close() return 0
def main(argv, outp=None): if outp == None: outp = s_output.OutPut() pars = argparse.ArgumentParser(prog='autodoc', description=descr) #pars.add_argument('--format', default='rst') pars.add_argument('--cortex', default='ram://', help='Cortex URL for model inspection') pars.add_argument( '--doc-model', action='store_true', default=False, help='Generate RST docs for the DataModel within a cortex') pars.add_argument('--savefile', default=None, help='Save output to the given file') opts = pars.parse_args(argv) if opts.savefile: fd = open(opts.savefile, 'wb') outp = s_output.OutPutFd(fd) core = s_cortex.openurl(opts.cortex) if opts.doc_model: forms = [] types = [] props = collections.defaultdict(list) for tufo in core.getTufosByProp('syn:type'): name = tufo[1].get('syn:type') info = s_tufo.props(tufo) types.append((name, info)) for tufo in core.getTufosByProp('syn:form'): name = tufo[1].get('syn:form') info = s_tufo.props(tufo) forms.append((name, info)) for tufo in core.getTufosByProp('syn:prop'): prop = tufo[1].get('syn:prop') form = tufo[1].get('syn:prop:form') info = s_tufo.props(tufo) props[form].append((prop, info)) types.sort() forms.sort() [v.sort() for v in props.values()] rst = RstHelp() rst.addHead('Synapse Data Model', lvl=0) rst.addHead('Types', lvl=1) for name, info in types: rst.addHead(name, lvl=2) inst = core.getTypeInst(name) ex = inst.get('ex') doc = inst.get('doc') if doc != None: rst.addLines(doc) bases = core.getTypeBases(name) rst.addLines('', 'Type Hierarchy: %s' % (' -> '.join(bases), ), '') if ex != None: #valu = core.getTypeParse(name,ex) #vrep = reprvalu(valu) rst.addLines('', 'Examples:', '') rst.addLines('- repr mode: %s' % (repr(ex), )) #rst.addLines('- system mode: %s' % (vrep,)) rst.addLines('') cons = [] xforms = [] if core.isSubType(name, 'str'): regex = inst.get('regex') if regex != None: cons.append('- regex: %s' % (regex, )) lower = inst.get('lower') if lower: xforms.append('- case: lower') restrip = inst.get('restrip') if restrip != None: xforms.append('- regex strip: %s' % (restrip, )) nullval = inst.get('nullval') if nullval != None: cons.append('- null value: %s' % (nullval, )) if core.isSubType(name, 'int'): minval = inst.get('min') if minval != None: cons.append('- min value: %d (0x%x)' % (minval, minval)) maxval = inst.get('max') if maxval != None: cons.append('- max value: %d (0x%x)' % (maxval, maxval)) ismin = inst.get('ismin') if ismin != None: xforms.append('- is minimum: True') ismax = inst.get('ismax') if ismax != None: xforms.append('- is maximum: True') if core.isSubType(name, 'sepr'): sep = inst.get('sep') fields = inst.get('fields') parts = [] for part in fields.split('|'): name, stype = part.split(',') parts.append(stype) seprs = sep.join(['<%s>' % p for p in parts]) rst.addLines('', 'Sepr Fields: %s' % (seprs, )) if cons: cons.append('') rst.addLines('', 'Type Constraints:', '', *cons) if xforms: xforms.append('') rst.addLines('', 'Type Transforms:', '', *xforms) rst.addHead('Forms', lvl=1) for name, info in forms: ftype = info.get('ptype', 'str') rst.addHead('%s = <%s>' % (name, ftype), lvl=2) doc = core.getPropInfo(name, 'doc') if doc != None: rst.addLines(doc) rst.addLines('', 'Properties:', '') for prop, pnfo in props.get(name, ()): # use the resolver funcs that will recurse upward pex = core.getPropInfo(prop, 'ex') pdoc = core.getPropInfo(prop, 'doc') ptype = pnfo.get('ptype') pline = '\t- %s = <%s>' % (prop, ptype) defval = pnfo.get('defval') if defval != None: pline += ' (default: %r)' % (defval, ) rst.addLines(pline) if pdoc: rst.addLines('\t\t- %s' % (pdoc, )) outp.printf(rst.getRstText()) return 0