def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds } if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split( bytestring(replace_gnu_args(self.args, **format_kwargs))) else: args = [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args ] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) logger.debug("process args: %s", args) return args
def format_args(self): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds, 'hostname': socket.gethostname(), 'sockets': self.watcher._get_sockets_fds()} if self.watcher is not None: format_kwargs['sockets'] = self.watcher._get_sockets_fds() for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split(bytestring(replace_gnu_args( self.args, **format_kwargs))) else: args = [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) for k, v in self.env.items(): self.env[k] = replace_gnu_args(v, **format_kwargs) logger.debug("process args: %s", args) return args
def resolve_name(import_name, silent=False): """Imports an object based on a string. This is useful if you want to use import paths as endpoints or something similar. An import path can be specified either in dotted notation (``xml.sax.saxutils.escape``) or with a colon as object delimiter (``xml.sax.saxutils:escape``). If `silent` is True the return value will be `None` if the import fails. :param import_name: the dotted name for the object to import. :param silent: if set to `True` import errors are ignored and `None` is returned instead. :return: imported object """ # force the import name to automatically convert to strings import_name = bytestring(import_name) try: if ':' in import_name: module, obj = import_name.split(':', 1) elif '.' in import_name: module, obj = import_name.rsplit('.', 1) else: return __import__(import_name) # __import__ is not able to handle unicode strings in the fromlist # if the module is a package try: return getattr(__import__(module, None, None, [obj]), obj) except (ImportError, AttributeError): # support importing modules not yet set up by the parent module # (or package for that matter) modname = module + '.' + obj __import__(modname) return sys.modules[modname] except ImportError as e: if not silent: raise_with_tb(ImportStringError(import_name, e))
def resolve_name(import_name, silent=False, reload=False): """Imports an object based on a string. This is useful if you want to use import paths as endpoints or something similar. An import path can be specified either in dotted notation (``xml.sax.saxutils.escape``) or with a colon as object delimiter (``xml.sax.saxutils:escape``). If `silent` is True the return value will be `None` if the import fails. :param import_name: the dotted name for the object to import. :param silent: if set to `True` import errors are ignored and `None` is returned instead. :param reload: if set to `True` modules that are already loaded will be reloaded :return: imported object """ # force the import name to automatically convert to strings import_name = bytestring(import_name) try: if ':' in import_name: module, obj = import_name.split(':', 1) elif '.' in import_name and import_name not in sys.modules: module, obj = import_name.rsplit('.', 1) else: module, obj = import_name, None # __import__ is not able to handle unicode strings in the fromlist mod = None # if the module is a package if reload and module in sys.modules: try: importlib.invalidate_caches() except Exception: pass try: mod = reload_module(sys.modules[module]) except Exception: pass if not mod: if not obj: return __import__(module) try: mod = __import__(module, None, None, [obj]) except ImportError: if ':' in import_name: raise return __import__(import_name) if not obj: return mod try: return getattr(mod, obj) except AttributeError: # support importing modules not yet set up by the parent module # (or package for that matter) if ':' in import_name: raise return __import__(import_name) except ImportError as e: if not silent: raise_with_tb(ImportStringError(import_name, e))
def notify_event(self, topic, msg): """Publish a message on the event publisher channel""" name = bytestring(self.res_name) multipart_msg = [b("watcher.%s.%s" % (name, topic)), json.dumps(msg)] if self.evpub_socket is not None and not self.evpub_socket.closed: self.evpub_socket.send_multipart(multipart_msg)
def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds } if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split( bytestring(replace_gnu_args(self.args, **format_kwargs))) else: args = [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args ] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) if self.shell: # subprocess.Popen(shell=True) implies that 1st arg is the # requested command, remaining args are applied to sh. args = [' '.join(quote(arg) for arg in args)] shell_args = format_kwargs.get('shell_args', None) if shell_args and is_win(): logger.warn( "shell_args won't apply for " "windows platforms: %s", shell_args) elif isinstance(shell_args, string_types): args += shlex.split( bytestring(replace_gnu_args(shell_args, **format_kwargs))) elif shell_args: args += [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in shell_args ] elif format_kwargs.get('shell_args', False): logger.warn( "shell_args is defined but won't be used " "in this context: %s", format_kwargs['shell_args']) logger.debug("process args: %s", args) return args
def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds} if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split(bytestring(replace_gnu_args( self.args, **format_kwargs))) else: args = [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) if self.shell: # subprocess.Popen(shell=True) implies that 1st arg is the # requested command, remaining args are applied to sh. args = [' '.join(quote(arg) for arg in args)] shell_args = format_kwargs.get('shell_args', None) if shell_args and is_win(): logger.warn("shell_args won't apply for " "windows platforms: %s", shell_args) elif isinstance(shell_args, string_types): args += shlex.split(bytestring(replace_gnu_args( shell_args, **format_kwargs))) elif shell_args: args += [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in shell_args] elif format_kwargs.get('shell_args', False): logger.warn("shell_args is defined but won't be used " "in this context: %s", format_kwargs['shell_args']) logger.debug("process args: %s", args) return args
def __init__(self, wid, cmd, args=None, working_dir=None, shell=False, uid=None, gid=None, env=None, rlimits=None, executable=None): self.wid = wid if working_dir is None: self.working_dir = get_working_dir() else: self.working_dir = working_dir self.shell = shell self.env = env if rlimits is not None: self.rlimits = rlimits else: self.rlimits = {} self.cmd = cmd.replace('$WID', str(self.wid)) if uid is None: self.uid = None else: self.uid = to_uid(uid) if gid is None: self.gid = None else: self.gid = to_gid(gid) def preexec_fn(): os.setsid() for limit, value in self.rlimits.items(): res = getattr(resource, 'RLIMIT_%s' % limit.upper(), None) if res is None: raise ValueError('unknown rlimit "%s"' % limit) # TODO(petef): support hard/soft limits resource.setrlimit(res, (value, value)) if self.gid: try: os.setgid(self.gid) except OverflowError: if not ctypes: raise # versions of python < 2.6.2 don't manage unsigned int for # groups like on osx or fedora os.setgid(-ctypes.c_int(-self.gid).value) if self.uid: os.setuid(self.uid) logger.debug('cmd: ' + cmd) logger.debug('args: ' + str(args)) if args is not None: if isinstance(args, str): args_ = shlex.split(bytestring(args)) else: args_ = args[:] args_ = shlex.split(bytestring(cmd)) + args_ else: args_ = shlex.split(bytestring(cmd)) logger.debug('Running %r' % ' '.join(args_)) self._worker = Popen(args_, cwd=self.working_dir, shell=self.shell, preexec_fn=preexec_fn, env=self.env, close_fds=True, stdout=PIPE, stderr=PIPE, executable=executable) self.started = time.time()
def __init__(self, wid, cmd, args=None, working_dir=None, shell=False, uid=None, gid=None, env=None, rlimits=None, executable=None): self.wid = wid if working_dir is None: self.working_dir = get_working_dir() else: self.working_dir = working_dir self.shell = shell self.env = env if rlimits is not None: self.rlimits = rlimits else: self.rlimits = {} self.cmd = cmd.replace('$WID', str(self.wid)) if uid is None: self.uid = None else: self.uid = to_uid(uid) if gid is None: self.gid = None else: self.gid = to_gid(gid) def preexec_fn(): os.setsid() for limit, value in self.rlimits.items(): res = getattr(resource, 'RLIMIT_%s' % limit.upper(), None) if res is None: raise ValueError('unknown rlimit "%s"' % limit) # TODO(petef): support hard/soft limits resource.setrlimit(res, (value, value)) if self.gid: try: os.setgid(self.gid) except OverflowError: if not ctypes: raise # versions of python < 2.6.2 don't manage unsigned int for # groups like on osx or fedora os.setgid(-ctypes.c_int(-self.gid).value) if self.uid: os.setuid(self.uid) logger.debug('cmd: ' + bytestring(cmd)) logger.debug('args: ' + str(args)) if args is not None: if isinstance(args, string_types): args_ = shlex.split(bytestring(args)) else: args_ = [bytestring(arg) for arg in args] args_ = shlex.split(bytestring(cmd)) + args_ else: args_ = shlex.split(bytestring(cmd)) logger.debug("process args: %s", args_) logger.debug('Running %r' % ' '.join(args_)) self._worker = Popen(args_, cwd=self.working_dir, shell=self.shell, preexec_fn=preexec_fn, env=self.env, close_fds=True, stdout=PIPE, stderr=PIPE, executable=executable) self.started = time.time()