def _onTeleOnMesg(self, sock, mesg):
try:
# set the socket tx method as the callback
jid = mesg[1].get('jid')
ons = mesg[1].get('ons')
name = mesg[1].get('name')
item = self.shared.get(name)
if item is None:
raise s_common.NoSuchObj(name=name)
user = sock.get('syn:user')
func = getattr(item, 'on', None)
if func is None:
return sock.tx(s_common.tufo('job:done', jid=jid, ret=False))
self._reqUserAllowed(user, 'tele:call', name, 'on')
for evnt, ontups in ons: # (<evnt>, ( (<iden>,<filt>), ... ))
onhelp, new = self._getOnHelp(name, evnt)
if new:
func(evnt, onhelp.dist)
for iden, filt in ontups:
onhelp.addOnInst(sock, iden, filt)
return sock.tx(s_common.tufo('job:done', jid=jid, ret=True))
except Exception as e:
sock.tx(s_common.tufo('job:done', jid=jid, **s_common.excinfo(e)))
def getModDef(name):
'''
Build a moddef tufo for the given module name.
Example:
moddef = getModDef('synapse.mindmeld')
'''
mod = s_dyndeps.getDynMod(name)
if mod is None:
return None
modpath = getattr(mod, '__file__', None)
if modpath is None:
return None
if modpath.endswith('.pyc'):
modpath = modpath[:-1]
modbase = os.path.basename(modpath)
modinfo = _getModInfo(modbase)
if modinfo is not None:
return s_common.tufo(name, path=modpath, **modinfo)
# hrm... what now smart guy?!?!
if name in sys.builtin_module_names:
return s_common.tufo(name, fmt='bin')
def _onTeleOffMesg(self, sock, mesg):
# set the socket tx method as the callback
try:
jid = mesg[1].get('jid')
evnt = mesg[1].get('evnt')
name = mesg[1].get('name')
iden = mesg[1].get('iden')
item = self.shared.get(name)
if item is None:
raise s_common.NoSuchObj(name=name)
onhelp, new = self._getOnHelp(name, evnt)
onhelp.delOnInst(sock, iden)
return sock.tx(s_common.tufo('job:done', jid=jid, ret=True))
except Exception as e:
errinfo = s_common.excinfo(e)
sock.tx(
s_common.tufo('job:done',
jid=jid,
err=errinfo.get('err'),
errinfo=errinfo))
def _onTelePushMesg(self, sock, mesg):
jid = mesg[1].get('jid')
name = mesg[1].get('name')
csides = mesg[1].get('csides')
reflect = mesg[1].get('reflect')
user = sock.get('syn:user')
if not self._isUserAllowed(user, 'tele:push:' + name):
return sock.tx(s_common.tufo('job:done', err='NoSuchRule',
jid=jid))
def onfini():
self.fire('tele:push:fini', name=name)
self.pushed.pop(name, None)
self.reflect.pop(name, None)
sock.onfini(onfini)
self.pushed[name] = sock
self.csides[name] = csides
self.reflect[name] = reflect
return sock.tx(s_common.tufo('job:done', jid=jid))
class PairType(s_types.DataType):
'''
silly type for representing a pair like "a!b".
it has subprops 'first' and 'second'.
'''
subprops = (
s_common.tufo('first', ptype='str'),
s_common.tufo('second', ptype='str'),
)
def norm(self, valu):
return self.chop(valu)[0]
def chop(self, valu):
if self.info.get('lower'):
valu = valu.lower()
first, _, second = valu.partition('!')
return valu, {
'first': first,
'second': second,
}
def parse(self, text):
return self.norm(text)
def repr(self, valu):
return valu
def _onTeleSynMesg(self, sock, mesg):
'''
Handle a telepath tele:syn message which is used to setup
a telepath session.
'''
jid = mesg[1].get('jid')
# pass / consume protocol version information
vers = mesg[1].get('vers', (0, 0))
name = mesg[1].get('name')
hisopts = mesg[1].get('opts', {})
if hisopts.get('sock:can:gzip'):
sock.set('sock:can:gzip', True)
if vers[0] != s_telepath.telever[0]:
info = s_common.errinfo(
'BadMesgVers',
'server %r != client %r' % (s_telepath.telever, vers))
return sock.tx(s_common.tufo('job:done', jid=jid, **info))
sess = None
iden = mesg[1].get('sess')
if iden is not None:
sess = self.getSessByIden(iden)
if sess is None:
sess = self.getNewSess()
ret = {
'sess': sess.iden,
'vers': s_telepath.telever,
'opts': {
'sock:can:gzip': True
},
}
if name is not None:
ret['csides'] = self.csides.get(name)
ret['reflect'] = self.reflect.get(name)
# send a nonce along for the ride in case
# they want to authenticate for the session
if not sess.get('user'):
nonce = s_common.guid()
ret['nonce'] = nonce
sess.put('nonce', nonce)
return sock.tx(s_common.tufo('job:done', jid=jid, ret=ret))
def _addWorkSlot(self):
iden = s_common.guid()
slot = s_common.tufo(iden, drone=self.iden)
self.slots[iden] = slot
self.slocs[iden] = {}
self.queen.addWorkSlot(slot)
def _onTeleRetnMesg(self, sock, mesg):
# tele:retn - used to pump a job:done to a client
suid = mesg[1].get('suid')
if suid is None:
return
dest = self.socks.get(suid)
if dest is None:
return
dest.tx(s_common.tufo('job:done', **mesg[1]))
def getPyStdLib():
'''
Get a {name:moddef} dict for python stdlib.
'''
global pymods
if pymods is None:
pylib = sysconfig.get_python_lib(standard_lib=True)
pymods = getModsByPath(pylib)
# get the compiled in modules
bins = sys.builtin_module_names
pymods.update({n: s_common.tufo(n, fmt='bin') for n in bins})
return pymods
def addMeldMod(self, name, byts, **modinfo):
'''
Add a MindMeld module by name and bytes.
byts = marshal.dumps( compile( sorc ) )
Note:
This API is mostly for use by routines like
addPySource.
'''
modinfo['fmt'] = 'pyc'
modinfo['bytes'] = byts
modinfo['pyver'] = majmin
self.info['modules'][name] = s_common.tufo(name, **modinfo)
def jobDoneMesg(job):
'''
Construct a job:done message for the given job.
Example:
def ondone(job):
otherguy.dist( jobDoneMesg(job) )
'''
info = {'jid': job[0], 'ret': job[1].get('ret')}
if job[1].get('err') is not None:
info['err'] = job[1].get('err'),
info['errmsg'] = job[1].get('errmsg'),
info['errfile'] = job[1].get('errfile'),
info['errline'] = job[1].get('errline'),
return s_common.tufo('job:done', **info)
def task(self, task, jid=None):
'''
Run the given task in the pool.
Example:
task = newtask( x.getFooByBar, bar )
pool.task(task, jid=None)
Notes:
* Specify jid=<iden> to generate job:done events.
'''
work = s_common.tufo(task, jid=jid)
with self._pool_lock:
if self.isfini:
raise s_common.IsFini(self.__class__.__name__)
# we're about to put work into the queue
# lets see if we should also fire another worker
# if there are available threads, no need to fire
if self._pool_avail != 0:
self.workq.put(work)
return
# got any breathing room?
if self._pool_maxsize > len(self._pool_threads):
self._fire_thread(self._run_work)
self.workq.put(work)
return
# got *all* the breathing room?
if self._pool_maxsize == -1:
self._fire_thread(self._run_work)
self.workq.put(work)
return
self.workq.put(work)
def _onTeleCallMesg(self, sock, mesg):
# tele:call - call a method on a shared object
jid = mesg[1].get('jid')
sid = mesg[1].get('sid')
# check if the socket knows about their auth
# ( most likely via SSL client cert )
user = sock.get('syn:user')
with s_scope.enter({
'dmon': self,
'sock': sock,
'syn:user': user,
'syn:auth': self.auth
}):
try:
name = mesg[1].get('name')
item = self.shared.get(name)
if item is None:
# is it a pushed object?
pushsock = self.pushed.get(name)
if pushsock is not None:
# pass along how to reply
mesg[1]['suid'] = sock.iden
return pushsock.tx(mesg)
raise s_common.NoSuchObj(name)
task = mesg[1].get('task')
meth, args, kwargs = task
self._reqUserAllowed(user, 'tele:call', name, meth)
func = getattr(item, meth, None)
if func is None:
raise s_common.NoSuchMeth(meth)
if getattr(func, '_tele_clientside', False):
name = s_reflect.getMethName(func)
raise s_common.TeleClientSide(name=name)
ret = func(*args, **kwargs)
# handle generator returns specially
if isinstance(ret, types.GeneratorType):
iden = s_common.guid()
txwait = threading.Event()
# start off set...
txwait.set()
self._dmon_yields.add(iden)
sock.tx(
s_common.tufo('tele:yield:init', jid=jid, iden=iden))
# FIXME opt
maxsize = 100000000
def ontxsize(m):
size = m[1].get('size')
if size >= maxsize:
txwait.clear()
else:
txwait.set()
try:
sock.onfini(txwait.set)
sock.on('sock:tx:size', ontxsize)
for item in ret:
txwait.wait()
# check if we woke due to fini
if sock.isfini:
break
sock.tx(
s_common.tufo('tele:yield:item',
iden=iden,
item=item))
if iden not in self._dmon_yields:
break
finally:
sock.off('sock:tx:size', ontxsize)
self._dmon_yields.discard(iden)
sock.tx(s_common.tufo('tele:yield:fini', iden=iden))
return
sock.tx(s_common.tufo('job:done', jid=jid, ret=ret))
except Exception as e:
sock.tx(
s_common.tufo('job:done', jid=jid, **s_common.excinfo(e)))
def getModsByPath(path, modtree=None):
'''
Return a list of (modname,info) tuples for a path entry.
Example:
for path in sys.path:
mods = getModsByPath(path)
dostuff(mods)
'''
path = os.path.abspath(path)
if modtree is None:
modtree = []
if not os.path.isdir(path):
raise s_common.NoSuchDir(path=path)
mods = {}
todo = [(path, modtree)]
while todo:
path, modtree = todo.pop()
pkgname = '.'.join(modtree)
for name in os.listdir(path):
if isSkipName(name):
continue
subbase = name.rsplit('.')[0]
subtree = modtree + [subbase]
subpath = os.path.join(path, name)
modname = '.'.join(subtree)
# check for a pkg dir...
if os.path.isdir(subpath):
pkgfile = os.path.join(subpath, '__init__.py')
if not os.path.isfile(pkgfile):
continue
# pkg dir found!
mods[modname] = s_common.tufo(modname,
fmt='src',
path=pkgfile,
pkg=True)
todo.append((subpath, subtree))
continue
modinfo = _getModInfo(name)
if modinfo is not None:
# fmt=None for unhandled module types
if not modinfo.get('fmt'):
continue
mods[modname] = s_common.tufo(modname, path=subpath, **modinfo)
continue
# add dat files to our pkg moddef
pmod = mods.get(pkgname)
if pmod is None:
continue
dats = pmod[1].get('dats')
if dats is None:
dats = {}
pmod[1]['dats'] = dats
dats[name] = subpath
return mods
import distutils.sysconfig as sysconfig
import synapse.common as s_common
import synapse.compat as s_compat
import synapse.dyndeps as s_dyndeps
import synapse.lib.tags as s_tags
'''
A set of utilities for locating/inspecting python modules.
'''
IMPORT_NAME = dis.opname.index('IMPORT_NAME')
IMPORT_FROM = dis.opname.index('IMPORT_FROM')
modtypes = (
s_common.tufo('.py', fmt='src'),
s_common.tufo('.pyd', fmt=None),
s_common.tufo('.pyo', fmt=None),
s_common.tufo('.pyc', fmt=None),
)
skippfx = ('.', )
skipsfx = ('.pyc', )
skipfiles = ('.', '..', '__init__.py', '__pycache__')
def isSkipName(name):
'''
Check if a file basename is a known "skipable".
'''
for pfx in skippfx:
