async def main(argv, outp=None):
if outp is None:
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
if opts.doc_model:
if opts.cortex:
async with await s_telepath.openurl(opts.cortex) as core:
rsttypes, rstforms = await docModel(outp, core)
else:
async with s_cortex.getTempCortex() as core:
rsttypes, rstforms = await docModel(outp, core)
if opts.savedir:
with open(s_common.genpath(opts.savedir, 'datamodel_types.rst'),
'wb') as fd:
fd.write(rsttypes.getRstText().encode())
with open(s_common.genpath(opts.savedir, 'datamodel_forms.rst'),
'wb') as fd:
fd.write(rstforms.getRstText().encode())
return 0
async def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
if opts.offset:
if len(opts.files) > 1:
outp.printf('Cannot start from a arbitrary offset for more than 1 file.')
return 1
outp.printf(f'Starting from offset [{opts.offset}] - it may take a while'
f' to get to that location in the input file.')
if opts.test:
async with s_cortex.getTempCortex(mods=opts.modules) as prox:
await addFeedData(prox, outp, opts.format, opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
elif opts.cortex:
async with await s_telepath.openurl(opts.cortex) as core:
await addFeedData(core, outp, opts.format, opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
else: # pragma: no cover
outp.printf('No valid options provided [%s]', opts)
return 1
return 0
async def main(argv, outp=None):
if outp is None:
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
if opts.doc_model:
if opts.cortex:
async with await s_telepath.openurl(opts.cortex) as core:
rsttypes, rstforms = await docModel(outp, core)
else:
async with s_cortex.getTempCortex() as core:
rsttypes, rstforms = await docModel(outp, core)
if opts.savedir:
with open(s_common.genpath(opts.savedir, 'datamodel_types.rst'), 'wb') as fd:
fd.write(rsttypes.getRstText().encode())
with open(s_common.genpath(opts.savedir, 'datamodel_forms.rst'), 'wb') as fd:
fd.write(rstforms.getRstText().encode())
if opts.doc_conf:
confdocs, cname = await docConfdefs(opts.doc_conf,
reflink=opts.doc_conf_reflink,
doc_title=opts.doc_conf_title,
)
if opts.savedir:
with open(s_common.genpath(opts.savedir, f'conf_{cname.lower()}.rst'), 'wb') as fd:
fd.write(confdocs.getRstText().encode())
if opts.doc_storm:
confdocs, svcname = await docStormsvc(opts.doc_storm)
if opts.savedir:
with open(s_common.genpath(opts.savedir, f'stormsvc_{svcname.lower()}.rst'), 'wb') as fd:
fd.write(confdocs.getRstText().encode())
if opts.doc_stormpkg:
pkgdocs, pkgname = await docStormpkg(opts.doc_stormpkg)
if opts.savedir:
with open(s_common.genpath(opts.savedir, f'stormpkg_{pkgname.lower()}.rst'), 'wb') as fd:
fd.write(pkgdocs.getRstText().encode())
if opts.doc_stormtypes:
libdocs, typedocs = await docStormTypes()
if opts.savedir:
with open(s_common.genpath(opts.savedir, f'stormtypes_libs.rst'), 'wb') as fd:
fd.write(libdocs.getRstText().encode())
with open(s_common.genpath(opts.savedir, f'stormtypes_prims.rst'), 'wb') as fd:
fd.write(typedocs.getRstText().encode())
return 0
async def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
if opts.offset:
if len(opts.files) > 1:
outp.printf(
'Cannot start from a arbitrary offset for more than 1 file.')
return 1
outp.printf(
f'Starting from offset [{opts.offset}] - it may take a while'
f' to get to that location in the input file.')
if opts.test:
async with s_cortex.getTempCortex(mods=opts.modules) as prox:
await addFeedData(prox,
outp,
opts.format,
opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
elif opts.cortex:
async with await s_telepath.openurl(opts.cortex) as core:
try:
s_version.reqVersion(core._getSynVers(), reqver)
except s_exc.BadVersion as e:
valu = s_version.fmtVersion(*e.get('valu'))
outp.printf(
f'Cortex version {valu} is outside of the feed tool supported range ({reqver}).'
)
outp.printf(
f'Please use a version of Synapse which supports {valu}; '
f'current version is {s_version.verstring}.')
return 1
await addFeedData(core,
outp,
opts.format,
opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
else: # pragma: no cover
outp.printf('No valid options provided [%s]', opts)
return 1
return 0
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
if opts.offset:
if len(opts.files) > 1:
outp.printf(
'Cannot start from a arbitrary offset for more than 1 file.')
return 1
outp.printf(
f'Starting from offset [{opts.offset}] - it may take a while'
f' to get to that location in the input file.')
if opts.test:
with s_cortex.getTempCortex(opts.modules) as prox:
addFeedData(prox,
outp,
opts.format,
opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
elif opts.cortex:
with s_telepath.openurl(opts.cortex) as core:
addFeedData(core,
outp,
opts.format,
opts.debug,
chunksize=opts.chunksize,
offset=opts.offset,
*opts.files)
else: # pragma: no cover
outp.printf('No valid options provided [%s]', opts)
return 1
return 0
async def main(argv, outp=s_output.stdout):
pars = makeargparser()
try:
opts = pars.parse_args(argv)
except s_exc.ParserExit as e:
return e.get('status')
with open(opts.stormfile, 'r', encoding='utf8') as fd:
text = fd.read()
def iterrows():
for path in opts.csvfiles:
with open(path, 'r', encoding='utf8') as fd:
if opts.csv_header:
fd.readline()
def genr():
for row in csv.reader(fd):
yield row
for rows in s_common.chunks(genr(), 1000):
yield rows
rowgenr = iterrows()
logfd = None
if opts.logfile is not None:
logfd = s_common.genfile(opts.logfile)
async def addCsvData(core):
newcount, nodecount = 0, 0
for rows in rowgenr:
stormopts = {
'vars': {'rows': rows},
}
async for mesg in core.storm(text, opts=stormopts):
if mesg[0] == 'node:add':
newcount += 1
elif mesg[0] == 'node':
nodecount += 1
elif mesg[0] == 'err' and not opts.debug:
outp.printf(repr(mesg))
if opts.debug:
outp.printf(repr(mesg))
if logfd is not None:
byts = json.dumps(mesg).encode('utf8')
logfd.write(byts + b'\n')
if opts.cli:
await s_cmdr.runItemCmdr(core, outp)
return newcount, nodecount
if opts.test:
async with s_cortex.getTempCortex() as core:
newcount, nodecount = await addCsvData(core)
else:
async with await s_telepath.openurl(opts.cortex) as core:
newcount, nodecount = await addCsvData(core)
if logfd is not None:
logfd.close()
outp.printf('%d nodes (%d created).' % (nodecount, newcount,))
async def main(argv, outp=s_output.stdout):
pars = makeargparser()
try:
opts = pars.parse_args(argv)
except s_exc.ParserExit as e:
return e.get('status')
with open(opts.stormfile, 'r', encoding='utf8') as fd:
text = fd.read()
if opts.export:
if not opts.cortex:
outp.printf('--export requires --cortex')
return -1
if len(opts.csvfiles) != 1:
outp.printf('--export requires exactly 1 csvfile')
return -1
path = s_common.genpath(opts.csvfiles[0])
outp.printf(f'Exporting CSV rows to: {path}')
async with await s_telepath.openurl(opts.cortex) as core:
with open(path, 'w') as fd:
wcsv = csv.writer(fd)
# prevent streaming nodes by limiting shown events
opts = {'show': ('csv:row', 'print', 'warn', 'err')}
count = 0
async for name, info in core.storm(text, opts=opts):
if name == 'csv:row':
count += 1
wcsv.writerow(info['row'])
continue
if name in ('init', 'fini'):
continue
outp.printf('%s: %r' % (name, info))
outp.printf(f'exported {count} csv rows.')
return
def iterrows():
for path in opts.csvfiles:
with open(path, 'r', encoding='utf8') as fd:
if opts.csv_header:
fd.readline()
def genr():
for row in csv.reader(fd):
yield row
for rows in s_common.chunks(genr(), 1000):
yield rows
rowgenr = iterrows()
logfd = None
if opts.logfile is not None:
logfd = s_common.genfile(opts.logfile)
logfd.seek(0, 2)
async def addCsvData(core):
newcount, nodecount = 0, 0
for rows in rowgenr:
stormopts = {
'vars': {
'rows': rows
},
}
async for mesg in core.storm(text, opts=stormopts):
if mesg[0] == 'node:add':
newcount += 1
elif mesg[0] == 'node':
nodecount += 1
elif mesg[0] == 'err' and not opts.debug:
outp.printf(repr(mesg))
elif mesg[0] == 'print':
outp.printf(mesg[1].get('mesg'))
if opts.debug:
outp.printf(repr(mesg))
if logfd is not None:
byts = json.dumps(mesg).encode('utf8')
logfd.write(byts + b'\n')
if opts.cli:
await s_cmdr.runItemCmdr(core, outp, True)
return newcount, nodecount
if opts.test:
async with s_cortex.getTempCortex() as core:
newcount, nodecount = await addCsvData(core)
else:
async with await s_telepath.openurl(opts.cortex) as core:
newcount, nodecount = await addCsvData(core)
if logfd is not None:
logfd.close()
outp.printf('%d nodes (%d created).' % (
nodecount,
newcount,
))
def main(argv, outp=s_output.stdout):
pars = makeargparser()
try:
opts = pars.parse_args(argv)
except s_exc.ParserExit as e:
return e.get('status')
with open(opts.stormfile, 'r', encoding='utf8') as fd:
text = fd.read()
def iterrows():
for path in opts.csvfiles:
with open(path, 'r', encoding='utf8') as fd:
if opts.csv_header:
fd.readline()
def genr():
for row in csv.reader(fd):
yield row
for rows in s_common.chunks(genr(), 1000):
yield rows
rowgenr = iterrows()
logfd = None
if opts.logfile is not None:
logfd = s_common.genfile(opts.logfile)
def addCsvData(core):
newcount, nodecount = 0, 0
for rows in rowgenr:
stormopts = {
'vars': {'rows': rows},
}
for mesg in core.storm(text, opts=stormopts):
if mesg[0] == 'node:add':
newcount += 1
elif mesg[0] == 'node':
nodecount += 1
elif mesg[0] == 'err' and not opts.debug:
outp.printf(repr(mesg))
if opts.debug:
outp.printf(repr(mesg))
if logfd is not None:
byts = json.dumps(mesg).encode('utf8')
logfd.write(byts + b'\n')
if opts.cli:
s_cmdr.runItemCmdr(core, outp)
return newcount, nodecount
if opts.test:
with s_cortex.getTempCortex() as core:
newcount, nodecount = addCsvData(core)
else:
with s_telepath.openurl(opts.cortex) as core:
newcount, nodecount = addCsvData(core)
if logfd is not None:
logfd.close()
outp.printf('%d nodes (%d created).' % (nodecount, newcount,))