def _onTeleOnMesg(self, sock, mesg): try: # set the socket tx method as the callback jid = mesg[1].get('jid') ons = mesg[1].get('ons') name = mesg[1].get('name') item = self.shared.get(name) if item is None: raise s_common.NoSuchObj(name=name) user = sock.get('syn:user') func = getattr(item, 'on', None) if func is None: return sock.tx(s_common.tufo('job:done', jid=jid, ret=False)) self._reqUserAllowed(user, 'tele:call', name, 'on') for evnt, ontups in ons: # (<evnt>, ( (<iden>,<filt>), ... )) onhelp, new = self._getOnHelp(name, evnt) if new: func(evnt, onhelp.dist) for iden, filt in ontups: onhelp.addOnInst(sock, iden, filt) return sock.tx(s_common.tufo('job:done', jid=jid, ret=True)) except Exception as e: sock.tx(s_common.tufo('job:done', jid=jid, **s_common.excinfo(e)))
def getModDef(name): ''' Build a moddef tufo for the given module name. Example: moddef = getModDef('synapse.mindmeld') ''' mod = s_dyndeps.getDynMod(name) if mod is None: return None modpath = getattr(mod, '__file__', None) if modpath is None: return None if modpath.endswith('.pyc'): modpath = modpath[:-1] modbase = os.path.basename(modpath) modinfo = _getModInfo(modbase) if modinfo is not None: return s_common.tufo(name, path=modpath, **modinfo) # hrm... what now smart guy?!?! if name in sys.builtin_module_names: return s_common.tufo(name, fmt='bin')
def _onTeleOffMesg(self, sock, mesg): # set the socket tx method as the callback try: jid = mesg[1].get('jid') evnt = mesg[1].get('evnt') name = mesg[1].get('name') iden = mesg[1].get('iden') item = self.shared.get(name) if item is None: raise s_common.NoSuchObj(name=name) onhelp, new = self._getOnHelp(name, evnt) onhelp.delOnInst(sock, iden) return sock.tx(s_common.tufo('job:done', jid=jid, ret=True)) except Exception as e: errinfo = s_common.excinfo(e) sock.tx( s_common.tufo('job:done', jid=jid, err=errinfo.get('err'), errinfo=errinfo))
def _onTelePushMesg(self, sock, mesg): jid = mesg[1].get('jid') name = mesg[1].get('name') csides = mesg[1].get('csides') reflect = mesg[1].get('reflect') user = sock.get('syn:user') if not self._isUserAllowed(user, 'tele:push:' + name): return sock.tx(s_common.tufo('job:done', err='NoSuchRule', jid=jid)) def onfini(): self.fire('tele:push:fini', name=name) self.pushed.pop(name, None) self.reflect.pop(name, None) sock.onfini(onfini) self.pushed[name] = sock self.csides[name] = csides self.reflect[name] = reflect return sock.tx(s_common.tufo('job:done', jid=jid))
class PairType(s_types.DataType): ''' silly type for representing a pair like "a!b". it has subprops 'first' and 'second'. ''' subprops = ( s_common.tufo('first', ptype='str'), s_common.tufo('second', ptype='str'), ) def norm(self, valu): return self.chop(valu)[0] def chop(self, valu): if self.info.get('lower'): valu = valu.lower() first, _, second = valu.partition('!') return valu, { 'first': first, 'second': second, } def parse(self, text): return self.norm(text) def repr(self, valu): return valu
def _onTeleSynMesg(self, sock, mesg): ''' Handle a telepath tele:syn message which is used to setup a telepath session. ''' jid = mesg[1].get('jid') # pass / consume protocol version information vers = mesg[1].get('vers', (0, 0)) name = mesg[1].get('name') hisopts = mesg[1].get('opts', {}) if hisopts.get('sock:can:gzip'): sock.set('sock:can:gzip', True) if vers[0] != s_telepath.telever[0]: info = s_common.errinfo( 'BadMesgVers', 'server %r != client %r' % (s_telepath.telever, vers)) return sock.tx(s_common.tufo('job:done', jid=jid, **info)) sess = None iden = mesg[1].get('sess') if iden is not None: sess = self.getSessByIden(iden) if sess is None: sess = self.getNewSess() ret = { 'sess': sess.iden, 'vers': s_telepath.telever, 'opts': { 'sock:can:gzip': True }, } if name is not None: ret['csides'] = self.csides.get(name) ret['reflect'] = self.reflect.get(name) # send a nonce along for the ride in case # they want to authenticate for the session if not sess.get('user'): nonce = s_common.guid() ret['nonce'] = nonce sess.put('nonce', nonce) return sock.tx(s_common.tufo('job:done', jid=jid, ret=ret))
def _addWorkSlot(self): iden = s_common.guid() slot = s_common.tufo(iden, drone=self.iden) self.slots[iden] = slot self.slocs[iden] = {} self.queen.addWorkSlot(slot)
def _onTeleRetnMesg(self, sock, mesg): # tele:retn - used to pump a job:done to a client suid = mesg[1].get('suid') if suid is None: return dest = self.socks.get(suid) if dest is None: return dest.tx(s_common.tufo('job:done', **mesg[1]))
def getPyStdLib(): ''' Get a {name:moddef} dict for python stdlib. ''' global pymods if pymods is None: pylib = sysconfig.get_python_lib(standard_lib=True) pymods = getModsByPath(pylib) # get the compiled in modules bins = sys.builtin_module_names pymods.update({n: s_common.tufo(n, fmt='bin') for n in bins}) return pymods
def addMeldMod(self, name, byts, **modinfo): ''' Add a MindMeld module by name and bytes. byts = marshal.dumps( compile( sorc ) ) Note: This API is mostly for use by routines like addPySource. ''' modinfo['fmt'] = 'pyc' modinfo['bytes'] = byts modinfo['pyver'] = majmin self.info['modules'][name] = s_common.tufo(name, **modinfo)
def jobDoneMesg(job): ''' Construct a job:done message for the given job. Example: def ondone(job): otherguy.dist( jobDoneMesg(job) ) ''' info = {'jid': job[0], 'ret': job[1].get('ret')} if job[1].get('err') is not None: info['err'] = job[1].get('err'), info['errmsg'] = job[1].get('errmsg'), info['errfile'] = job[1].get('errfile'), info['errline'] = job[1].get('errline'), return s_common.tufo('job:done', **info)
def task(self, task, jid=None): ''' Run the given task in the pool. Example: task = newtask( x.getFooByBar, bar ) pool.task(task, jid=None) Notes: * Specify jid=<iden> to generate job:done events. ''' work = s_common.tufo(task, jid=jid) with self._pool_lock: if self.isfini: raise s_common.IsFini(self.__class__.__name__) # we're about to put work into the queue # lets see if we should also fire another worker # if there are available threads, no need to fire if self._pool_avail != 0: self.workq.put(work) return # got any breathing room? if self._pool_maxsize > len(self._pool_threads): self._fire_thread(self._run_work) self.workq.put(work) return # got *all* the breathing room? if self._pool_maxsize == -1: self._fire_thread(self._run_work) self.workq.put(work) return self.workq.put(work)
def _onTeleCallMesg(self, sock, mesg): # tele:call - call a method on a shared object jid = mesg[1].get('jid') sid = mesg[1].get('sid') # check if the socket knows about their auth # ( most likely via SSL client cert ) user = sock.get('syn:user') with s_scope.enter({ 'dmon': self, 'sock': sock, 'syn:user': user, 'syn:auth': self.auth }): try: name = mesg[1].get('name') item = self.shared.get(name) if item is None: # is it a pushed object? pushsock = self.pushed.get(name) if pushsock is not None: # pass along how to reply mesg[1]['suid'] = sock.iden return pushsock.tx(mesg) raise s_common.NoSuchObj(name) task = mesg[1].get('task') meth, args, kwargs = task self._reqUserAllowed(user, 'tele:call', name, meth) func = getattr(item, meth, None) if func is None: raise s_common.NoSuchMeth(meth) if getattr(func, '_tele_clientside', False): name = s_reflect.getMethName(func) raise s_common.TeleClientSide(name=name) ret = func(*args, **kwargs) # handle generator returns specially if isinstance(ret, types.GeneratorType): iden = s_common.guid() txwait = threading.Event() # start off set... txwait.set() self._dmon_yields.add(iden) sock.tx( s_common.tufo('tele:yield:init', jid=jid, iden=iden)) # FIXME opt maxsize = 100000000 def ontxsize(m): size = m[1].get('size') if size >= maxsize: txwait.clear() else: txwait.set() try: sock.onfini(txwait.set) sock.on('sock:tx:size', ontxsize) for item in ret: txwait.wait() # check if we woke due to fini if sock.isfini: break sock.tx( s_common.tufo('tele:yield:item', iden=iden, item=item)) if iden not in self._dmon_yields: break finally: sock.off('sock:tx:size', ontxsize) self._dmon_yields.discard(iden) sock.tx(s_common.tufo('tele:yield:fini', iden=iden)) return sock.tx(s_common.tufo('job:done', jid=jid, ret=ret)) except Exception as e: sock.tx( s_common.tufo('job:done', jid=jid, **s_common.excinfo(e)))
def getModsByPath(path, modtree=None): ''' Return a list of (modname,info) tuples for a path entry. Example: for path in sys.path: mods = getModsByPath(path) dostuff(mods) ''' path = os.path.abspath(path) if modtree is None: modtree = [] if not os.path.isdir(path): raise s_common.NoSuchDir(path=path) mods = {} todo = [(path, modtree)] while todo: path, modtree = todo.pop() pkgname = '.'.join(modtree) for name in os.listdir(path): if isSkipName(name): continue subbase = name.rsplit('.')[0] subtree = modtree + [subbase] subpath = os.path.join(path, name) modname = '.'.join(subtree) # check for a pkg dir... if os.path.isdir(subpath): pkgfile = os.path.join(subpath, '__init__.py') if not os.path.isfile(pkgfile): continue # pkg dir found! mods[modname] = s_common.tufo(modname, fmt='src', path=pkgfile, pkg=True) todo.append((subpath, subtree)) continue modinfo = _getModInfo(name) if modinfo is not None: # fmt=None for unhandled module types if not modinfo.get('fmt'): continue mods[modname] = s_common.tufo(modname, path=subpath, **modinfo) continue # add dat files to our pkg moddef pmod = mods.get(pkgname) if pmod is None: continue dats = pmod[1].get('dats') if dats is None: dats = {} pmod[1]['dats'] = dats dats[name] = subpath return mods
import distutils.sysconfig as sysconfig import synapse.common as s_common import synapse.compat as s_compat import synapse.dyndeps as s_dyndeps import synapse.lib.tags as s_tags ''' A set of utilities for locating/inspecting python modules. ''' IMPORT_NAME = dis.opname.index('IMPORT_NAME') IMPORT_FROM = dis.opname.index('IMPORT_FROM') modtypes = ( s_common.tufo('.py', fmt='src'), s_common.tufo('.pyd', fmt=None), s_common.tufo('.pyo', fmt=None), s_common.tufo('.pyc', fmt=None), ) skippfx = ('.', ) skipsfx = ('.pyc', ) skipfiles = ('.', '..', '__init__.py', '__pycache__') def isSkipName(name): ''' Check if a file basename is a known "skipable". ''' for pfx in skippfx: