def main(argv, outp=s_output.stdout):
argp = getArgParser()
opts = argp.parse_args(argv)
s_common.setlogging(logger, opts.log_level)
dirn = s_common.gendir(opts.dmonpath)
outp.printf(f'Beginning dmon from {dirn}')
# since we're dmon main, register our certdir as global
certdir = s_common.gendir(dirn, 'certs')
# set the default certdir to the dmon certdir
s_certdir.defdir = certdir
path = os.path.join(dirn, 'dmon.yaml')
if not os.path.isfile(path):
with open(path, 'wb') as fd:
fd.write(dmonyaml.encode('utf8'))
dmon = None
try:
dmon = s_glob.sync(s_daemon.Daemon.anit(opts.dmonpath))
host, port = dmon.addr
outp.printf(f'dmon listening on: {host}:{port}')
dmon.main()
finally:
if dmon is not None:
s_glob.sync(dmon.fini())
def __iter__(self):
try:
while not self.isfini:
for retn in s_glob.sync(self.queue.slice()):
if retn is None:
return
yield s_common.result(retn)
finally:
s_glob.sync(self.fini())
def __iter__(self):
try:
while not self.isfini:
for retn in s_glob.sync(self.queue.slice()):
if retn is None:
return
yield s_common.result(retn)
finally:
s_glob.sync(self.fini())
def test_glob_sync(self):
async def afoo():
return 42
retn = s_glob.sync(afoo())
self.eq(retn, 42)
def __iter__(self):
try:
while True:
item = s_glob.sync(self.genr.__anext__())
yield item
except StopAsyncIteration:
return
except GeneratorExit:
# Raised if a synchronous consumer exited an iterator early.
# Signal the generator to close down.
s_glob.sync(self.genr.aclose())
raise
def main(argv, outp=s_output.stdout): # pragma: no cover
teleport = os.getenv('SYN_CORTEX_PORT', '27492')
telehost = os.getenv('SYN_CORTEX_HOST', '127.0.0.1')
# httpport = os.getenv('SYN_CORTEX_HTTP_PORT', '80')
# httphost = os.getenv('SYN_CORTEX_HTTP_HOST', '127.0.0.1')
# httpsport = os.getenv('SYN_CORTEX_HTTPS_PORT', '443')
# httpshost = os.getenv('SYN_CORTEX_HTTPS_HOST', '127.0.0.1')
pars = argparse.ArgumentParser(prog='synapse.servers.cortex')
pars.add_argument('--port',
default=teleport,
help='The TCP port to bind for telepath.')
pars.add_argument('--host',
default=telehost,
help='The host address to bind telepath.')
pars.add_argument('coredir',
help='The directory for the cortex to use for storage.')
opts = pars.parse_args(argv)
s_common.setlogging(logger)
dmon = s_glob.sync(mainopts(opts, outp=outp))
return dmon.main()
def block_processing(evt1, evt2):
'''
Function to make a base and call main(). Used as a Process target.
Args:
evt1 (multiprocessing.Event): event to twiddle
evt2 (multiprocessing.Event): event to twiddle
'''
try:
base = s_glob.sync(s_base.Base.anit())
async def onMain(mesg):
evt1.set()
await base.fini()
def onFini():
evt2.set()
base.on('ebus:main', onMain)
base.onfini(onFini)
base.main()
finally:
sys.exit(137)
def main(self):
'''
Helper function to block until shutdown ( and handle ctrl-c and SIGTERM).
Examples:
Run a base, wait until main() has returned, then do other stuff::
foo = Base()
foo.main()
dostuff()
Notes:
This fires a 'ebus:main' event prior to entering the waitfini() loop.
Returns:
None
'''
doneevent = threading.Event()
self.onfini(doneevent.set)
async def sighandler():
print('Caught SIGTERM, shutting down')
await self.fini()
def handler():
asyncio.run_coroutine_threadsafe(sighandler(), loop=self.loop)
try:
self.loop.add_signal_handler(signal.SIGTERM, handler)
except Exception as e: # pragma: no cover
logger.exception('Unable to register SIGTERM handler.')
async def asyncmain():
await self.fire('ebus:main')
asyncio.run_coroutine_threadsafe(asyncmain(), loop=self.loop)
try:
doneevent.wait()
except KeyboardInterrupt as e:
print('ctrl-c caught: shutting down')
finally:
# Avoid https://bugs.python.org/issue34680 by removing handler before closing down
self.loop.remove_signal_handler(signal.SIGTERM)
s_glob.sync(self.fini())
def __iter__(self):
try:
while True:
item = s_glob.sync(self.genr.__anext__())
yield item
except StopAsyncIteration as e:
return
def __iter__(self):
try:
while True:
item = s_glob.sync(self.genr.__anext__())
yield item
except StopAsyncIteration:
return
def getTempCortex(mods=None):
'''
Get a proxy to a cortex backed by a temporary directory.
Args:
mods (list): A list of modules which are loaded into the cortex.
Notes:
The cortex and temporary directory are town down on exit.
This should only be called from synchronous code.
Returns:
Proxy to the cortex.
'''
with s_common.getTempDir() as dirn:
with s_coro.AsyncToSyncCMgr(s_glob.sync, Cortex.anit(dirn)) as core:
if mods:
for mod in mods:
s_glob.sync(core.loadCoreModule(mod))
with s_coro.AsyncToSyncCMgr(core.getLocalProxy) as prox:
yield prox
def testmain():
pconf = {'user': '******', 'passwd': 'root'}
with self.getTestProxy(dmon, 'core', **pconf) as core:
s_glob.sync(self.addCreatorDeleterRoles(core))
core.addUserRole('root', 'creator')
host, port = dmon.addr
curl = f'tcp://*****:*****@{host}:{port}/core'
with self.getTestDir() as dirn:
mpkfp = s_common.genpath(dirn, 'podes.mpk')
with s_common.genfile(mpkfp) as fd:
for i in range(20):
pode = (('testint', i), {})
fd.write(s_msgpack.en(pode))
argv = [
'--cortex', curl, '--format', 'syn.nodes', '--modules',
'synapse.tests.utils.TestModule', '--chunksize', '4',
'--offset', '15', mpkfp
]
outp = self.getTestOutp()
self.eq(s_feed.main(argv, outp=outp), 0)
with self.getTestProxy(dmon, 'core', **pconf) as core:
self.len(8, list(core.eval('testint')))
# Sad path catch
outp = self.getTestOutp()
argv.append(mpkfp)
self.eq(s_feed.main(argv, outp=outp), 1)
self.true(
outp.expect(
'Cannot start from a arbitrary offset for more than 1 file.'
))
def main(argv): # pragma: no cover
if len(argv) != 2:
print('usage: python -m synapse.tools.cmdr <url>')
return -1
with s_telepath.openurl(argv[1]) as item:
cmdr = s_glob.sync(s_cmdr.getItemCmdr(item))
# This causes a dropped connection to the cmdr'd item to
# cause cmdr to exit. We can't safely test this in CI since
# the fini handler sends a SIGINT to mainthread; which can
# be problematic for test runners.
cmdr.finikill = True
cmdr.runCmdLoop()
cmdr.finikill = False
def test_telepath_sync_genr_break(self):
try:
acm = self.getTestCoreAndProxy()
core, proxy = s_glob.sync(acm.__aenter__())
form = 'test:int'
q = '[' + ' '.join([f'{form}={i}' for i in range(10)]) + ' ]'
# This puts a link into the link pool
podes = list(proxy.eval(q))
self.len(10, podes)
evt = threading.Event()
# Get the link from the pool, add the fini callback and put it back
link = s_glob.sync(proxy.getPoolLink())
link.onfini(evt.set)
s_glob.sync(proxy._putPoolLink(link))
q = f'{form} | sleep 0.1'
# Break from the generator right away, causing a
# GeneratorExit in the GenrHelp object __iter__ method.
pode = None
for pode in proxy.eval(q):
break
# Ensure the query did yield an object
self.nn(pode)
# Ensure the link we have a reference too was torn down
self.true(evt.wait(4))
self.true(link.isfini)
finally:
s_glob.sync(acm.__aexit__(None, None, None))
def __iter__(self):
genr = s_glob.sync(self.proxy.task(self.todo, name=self.share))
for item in genr:
yield item
muxp.add_argument('--deluser', action='store_true', help='Delete the named user to the cortex.')
muxp.add_argument('--delrole', action='store_true', help='Delete the named role to the cortex.')
muxp.add_argument('--admin', action='store_true', help='Grant admin powers to the user/role.')
muxp.add_argument('--noadmin', action='store_true', help='Revoke admin powers from the user/role.')
muxp.add_argument('--lock', action='store_true', help='Lock the user account.')
muxp.add_argument('--unlock', action='store_true', help='Unlock the user account.')
muxp.add_argument('--passwd', help='Set the user password.')
muxp.add_argument('--grant', help='Grant the specified role to the user.')
muxp.add_argument('--revoke', help='Grant the specified role to the user.')
muxp.add_argument('--addrule', help='Add the given rule to the user/role.')
muxp.add_argument('--delrule', type=int, help='Delete the given rule number from the user/role.')
pars_mod.add_argument('--object', type=str, help='The iden of the object to which to apply the new rule. Only '
'supported on Cells running Synapse >= 0.1.33.')
pars_mod.add_argument('name', help='The user/role to modify.')
pars_mod.set_defaults(func=handleModify)
return pars
async def _main(): # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
return await main(sys.argv[1:])
if __name__ == '__main__': # pragma: no cover
sys.exit(s_glob.sync(_main()))
def __enter__(self):
return s_glob.sync(self.amgr.__aenter__())
def __exit__(self, *args):
return s_glob.sync(self.amgr.__aexit__(*args))
def main(argv, outp=None):
if outp is None: # pragma: no cover
outp = s_output.OutPut()
pars = makeargparser()
opts = pars.parse_args(argv)
axon = s_telepath.openurl(opts.axon)
core = None
if opts.cortex:
core = s_telepath.openurl(opts.cortex)
tags = {}
if opts.tags:
for tag in opts.tags.split(','):
tags[tag] = (None, None)
if tags:
outp.printf('adding tags: %r' % (list(tags.keys())))
filepaths = set()
for item in opts.filenames:
paths = glob.glob(item, recursive=opts.recursive)
if not paths:
outp.printf(f'filepath does not contain any files: {item}')
continue
filepaths.update([path for path in paths if os.path.isfile(path)])
for path in filepaths:
bname = os.path.basename(path)
hset = s_hashset.HashSet()
with s_common.reqfile(path) as fd:
hset.eatfd(fd)
fhashes = {htyp: hasher.hexdigest() for htyp, hasher in hset.hashes}
sha256 = fhashes.get('sha256')
bsha256 = s_common.uhex(sha256)
if not axon.has(bsha256):
with axon.upload() as upfd:
with s_common.genfile(path) as fd:
for byts in s_common.iterfd(fd):
upfd.write(byts)
size, hashval = upfd.save()
if hashval != bsha256: # pragma: no cover
raise s_exc.SynErr(mesg='hashes do not match',
ehash=s_common.ehex(hashval),
ahash=hashval)
outp.printf(f'Uploaded [{bname}] to axon')
else:
outp.printf(f'Axon already had [{bname}]')
if core:
pnode = (('file:bytes', f'sha256:{sha256}'), {
'props': {
'md5': fhashes.get('md5'),
'sha1': fhashes.get('sha1'),
'sha256': fhashes.get('sha256'),
'size': hset.size,
'name': bname,
},
'tags': tags,
})
node = list(core.addNodes([pnode]))[0]
iden = node[0][1]
size = node[1]['props']['size']
name = node[1]['props']['name']
mesg = f'file: {bname} ({size}) added to core ({iden}) as {name}'
outp.printf(mesg)
s_glob.sync(axon.fini())
if core:
s_glob.sync(core.fini())
return 0
def __iter__(self):
genr = s_glob.sync(self.proxy.task(self.todo, name=self.share))
for item in genr:
yield item
def test_glob_sync(self):
async def afoo():
return 42
retn = s_glob.sync(afoo())
self.eq(retn, 42)
muxp.add_argument('--adduser', action='store_true', help='Add the named user to the cortex.')
muxp.add_argument('--addrole', action='store_true', help='Add the named role to the cortex.')
muxp.add_argument('--deluser', action='store_true', help='Delete the named user to the cortex.')
muxp.add_argument('--delrole', action='store_true', help='Delete the named role to the cortex.')
muxp.add_argument('--admin', action='store_true', help='Grant admin powers to the user/role.')
muxp.add_argument('--noadmin', action='store_true', help='Revoke admin powers from the user/role.')
muxp.add_argument('--lock', action='store_true', help='Lock the user account.')
muxp.add_argument('--unlock', action='store_true', help='Unlock the user account.')
muxp.add_argument('--passwd', help='Set the user password.')
muxp.add_argument('--grant', help='Grant the specified role to the user.')
muxp.add_argument('--revoke', help='Grant the specified role to the user.')
muxp.add_argument('--addrule', help='Add the given rule to the user/role.')
muxp.add_argument('--delrule', type=int, help='Delete the given rule number from the user/role.')
pars_mod.add_argument('name', help='The user/role to modify.')
pars_mod.set_defaults(func=handleModify)
return pars
async def _main(): # pragma: no cover
s_common.setlogging(logger, 'DEBUG')
return await main(sys.argv[1:])
if __name__ == '__main__': # pragma: no cover
sys.exit(s_glob.sync(_main()))
